* [PATCH v3 1/3] bitbake: Update to 1.50.5 release
2022-04-22 9:05 [PATCH v3 0/3] bump bitbake and wic for python 3.10 support Henning Schild
@ 2022-04-22 9:06 ` Henning Schild
2022-04-22 9:06 ` [PATCH v3 2/3] wic: align our fork of common.wks.inc with wic version we carry Henning Schild
` (3 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Henning Schild @ 2022-04-22 9:06 UTC (permalink / raw)
To: isar-users; +Cc: Florian Bezdeka, Felix Moessbauer, Henning Schild
Update bitbake to the latest release in 1.50 branch. This release is
tagged to the commit ID aaa7f7af23d5f89fe4a5ed48c57ea3dfca07c79d
in the bitbake upstream.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
bitbake/lib/bb/cache.py | 3 +-
bitbake/lib/bb/cooker.py | 30 ++++++++++++++--
bitbake/lib/bb/data_smart.py | 4 +--
bitbake/lib/bb/fetch2/__init__.py | 4 +++
bitbake/lib/bb/fetch2/perforce.py | 2 +-
bitbake/lib/bb/fetch2/wget.py | 2 +-
bitbake/lib/bb/persist_data.py | 5 +--
bitbake/lib/bb/process.py | 2 +-
bitbake/lib/bb/runqueue.py | 34 +++++++++---------
bitbake/lib/bb/server/process.py | 2 +-
bitbake/lib/bb/tests/fetch.py | 35 ++++++++++---------
bitbake/lib/bb/utils.py | 13 +++++--
bitbake/lib/hashserv/server.py | 4 +--
bitbake/lib/toaster/tests/builds/buildtest.py | 2 +-
14 files changed, 89 insertions(+), 53 deletions(-)
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 27eb271798e8..5f9c0a779d75 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -19,7 +19,8 @@
import os
import logging
import pickle
-from collections import defaultdict, Mapping
+from collections import defaultdict
+from collections.abc import Mapping
import bb.utils
from bb import PrefixLoggerAdapter
import re
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 89f1fad08310..c946800a8c62 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -388,12 +388,22 @@ class BBCooker:
# Create a new hash server bound to a unix domain socket
if not self.hashserv:
dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+ upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
+ if upstream:
+ import socket
+ try:
+ sock = socket.create_connection(upstream.split(":"), 5)
+ sock.close()
+ except socket.error as e:
+ bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
+ % (upstream, repr(e)))
+
self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
self.hashserv = hashserv.create_server(
self.hashservaddr,
dbfile,
sync=False,
- upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None,
+ upstream=upstream,
)
self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
self.hashserv.process.start()
@@ -805,7 +815,9 @@ class BBCooker:
for dep in rq.rqdata.runtaskentries[tid].depends:
(depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
- depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
+ if depmc:
+ depmc = "mc:" + depmc + ":"
+ depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
if taskfn not in seen_fns:
seen_fns.append(taskfn)
packages = []
@@ -2204,21 +2216,33 @@ class CookerParser(object):
yield not cached, mc, infos
def parse_generator(self):
- while True:
+ empty = False
+ while self.processes or not empty:
+ for process in self.processes.copy():
+ if not process.is_alive():
+ process.join()
+ self.processes.remove(process)
+
if self.parsed >= self.toparse:
break
try:
result = self.result_queue.get(timeout=0.25)
except queue.Empty:
+ empty = True
pass
else:
+ empty = False
value = result[1]
if isinstance(value, BaseException):
raise value
else:
yield result
+ if not (self.parsed >= self.toparse):
+ raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
+
+
def parse_next(self):
result = []
parsed = None
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 8291ca65e309..65857a9c7941 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -17,7 +17,7 @@ BitBake build tools.
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
import copy, re, sys, traceback
-from collections import MutableMapping
+from collections.abc import MutableMapping
import logging
import hashlib
import bb, bb.codeparser
@@ -403,7 +403,7 @@ class DataSmart(MutableMapping):
s = __expand_python_regexp__.sub(varparse.python_sub, s)
except SyntaxError as e:
# Likely unmatched brackets, just don't expand the expression
- if e.msg != "EOL while scanning string literal":
+ if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
raise
if s == olds:
break
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index dbf8b50e68a7..1005ec10c639 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -430,6 +430,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
uri_replace_decoded = list(decodeurl(uri_replace))
logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
result_decoded = ['', '', '', '', '', {}]
+ # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params
for loc, i in enumerate(uri_find_decoded):
result_decoded[loc] = uri_decoded[loc]
regexp = i
@@ -449,6 +450,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
for l in replacements:
uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
result_decoded[loc][k] = uri_replace_decoded[loc][k]
+ elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
+ # User/password in the replacement is just a straight replacement
+ result_decoded[loc] = uri_replace_decoded[loc]
elif (re.match(regexp, uri_decoded[loc])):
if not uri_replace_decoded[loc]:
result_decoded[loc] = ""
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index e2a41a4a1287..3b6fa4b1ec9a 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -134,7 +134,7 @@ class Perforce(FetchMethod):
ud.setup_revisions(d)
- ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision))
+ ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
def _buildp4command(self, ud, d, command, depot_filename=None):
"""
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 784df70c9f62..7fa2a87ffde5 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -322,7 +322,7 @@ class Wget(FetchMethod):
except (TypeError, ImportError, IOError, netrc.NetrcParseError):
pass
- with opener.open(r) as response:
+ with opener.open(r, timeout=30) as response:
pass
except urllib.error.URLError as e:
if try_again:
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
index c6a209fb3fc1..6f32d81afe80 100644
--- a/bitbake/lib/bb/persist_data.py
+++ b/bitbake/lib/bb/persist_data.py
@@ -12,6 +12,7 @@ currently, providing a key/value store accessed by 'domain'.
#
import collections
+import collections.abc
import contextlib
import functools
import logging
@@ -19,7 +20,7 @@ import os.path
import sqlite3
import sys
import warnings
-from collections import Mapping
+from collections.abc import Mapping
sqlversion = sqlite3.sqlite_version_info
if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -29,7 +30,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
logger = logging.getLogger("BitBake.PersistData")
@functools.total_ordering
-class SQLTable(collections.MutableMapping):
+class SQLTable(collections.abc.MutableMapping):
class _Decorators(object):
@staticmethod
def retry(*, reconnect=True):
diff --git a/bitbake/lib/bb/process.py b/bitbake/lib/bb/process.py
index d5a1775fcec0..af5d804a1d59 100644
--- a/bitbake/lib/bb/process.py
+++ b/bitbake/lib/bb/process.py
@@ -60,7 +60,7 @@ class Popen(subprocess.Popen):
"close_fds": True,
"preexec_fn": subprocess_setup,
"stdout": subprocess.PIPE,
- "stderr": subprocess.STDOUT,
+ "stderr": subprocess.PIPE,
"stdin": subprocess.PIPE,
"shell": False,
}
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 10511a09dc1c..cd10da8b3a6f 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -926,38 +926,36 @@ class RunQueueData:
#
# Once all active tasks are marked, prune the ones we don't need.
- delcount = {}
- for tid in list(self.runtaskentries.keys()):
- if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
- del self.runtaskentries[tid]
-
# Handle --runall
if self.cooker.configuration.runall:
# re-run the mark_active and then drop unused tasks from new list
+ reduced_tasklist = set(self.runtaskentries.keys())
+ for tid in list(self.runtaskentries.keys()):
+ if tid not in runq_build:
+ reduced_tasklist.remove(tid)
runq_build = {}
for task in self.cooker.configuration.runall:
if not task.startswith("do_"):
task = "do_{0}".format(task)
runall_tids = set()
- for tid in list(self.runtaskentries):
+ for tid in reduced_tasklist:
wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
- if wanttid in delcount:
- self.runtaskentries[wanttid] = delcount[wanttid]
if wanttid in self.runtaskentries:
runall_tids.add(wanttid)
for tid in list(runall_tids):
- mark_active(tid,1)
+ mark_active(tid, 1)
if self.cooker.configuration.force:
invalidate_task(tid, False)
- for tid in list(self.runtaskentries.keys()):
- if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
- del self.runtaskentries[tid]
+ delcount = set()
+ for tid in list(self.runtaskentries.keys()):
+ if tid not in runq_build:
+ delcount.add(tid)
+ del self.runtaskentries[tid]
+ if self.cooker.configuration.runall:
if len(self.runtaskentries) == 0:
bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
@@ -971,16 +969,16 @@ class RunQueueData:
for task in self.cooker.configuration.runonly:
if not task.startswith("do_"):
task = "do_{0}".format(task)
- runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
+ runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task]
- for tid in list(runonly_tids):
- mark_active(tid,1)
+ for tid in runonly_tids:
+ mark_active(tid, 1)
if self.cooker.configuration.force:
invalidate_task(tid, False)
for tid in list(self.runtaskentries.keys()):
if tid not in runq_build:
- delcount[tid] = self.runtaskentries[tid]
+ delcount.add(tid)
del self.runtaskentries[tid]
if len(self.runtaskentries) == 0:
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 07bb785a1822..fcdce19717d2 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -659,7 +659,7 @@ class BBUIEventQueue:
self.reader = ConnectionReader(readfd)
self.t = threading.Thread()
- self.t.setDaemon(True)
+ self.t.daemon = True
self.t.run = self.startCallbackHandler
self.t.start()
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index f5d557e8c000..3b64584da0e9 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -431,6 +431,10 @@ class MirrorUriTest(FetcherTest):
("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
: "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+ ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
+ : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+
+
#Renaming files doesn't work
#("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
#("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
@@ -491,7 +495,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
super(GitDownloadDirectoryNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_dir = "git.openembedded.org.bitbake"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_dir = "github.com.openembedded.bitbake.git"
self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
@@ -539,7 +543,7 @@ class TarballNamingTest(FetcherTest):
super(TarballNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
@@ -573,7 +577,7 @@ class GitShallowTarballNamingTest(FetcherTest):
super(GitShallowTarballNamingTest, self).setUp()
self.recipe_url = "git://git.openembedded.org/bitbake"
self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
- self.mirror_url = "git://github.com/openembedded/bitbake.git"
+ self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -985,7 +989,7 @@ class FetcherNetworkTest(FetcherTest):
def test_git_submodule_dbus_broker(self):
# The following external repositories have show failures in fetch and unpack operations
# We want to avoid regressions!
- url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
+ url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1001,7 +1005,7 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_CLI11(self):
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1016,12 +1020,12 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_update_CLI11(self):
""" Prevent regression on update detection not finding missing submodule, or modules without needed commits """
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# CLI11 that pulls in a newer nlohmann-json
- url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1035,7 +1039,7 @@ class FetcherNetworkTest(FetcherTest):
@skipIfNoNetwork()
def test_git_submodule_aktualizr(self):
- url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+ url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1055,7 +1059,7 @@ class FetcherNetworkTest(FetcherTest):
""" Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
# This repository also has submodules where the module (name), path and url do not align
- url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+ url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699"
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
@@ -1113,7 +1117,7 @@ class SVNTest(FetcherTest):
bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
# Github will emulate SVN. Use this to check if we're downloding...
- bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
+ bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .",
cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
bb.process.run("svn commit --non-interactive -m 'Add external'",
cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1231,7 +1235,7 @@ class FetchLatestVersionTest(FetcherTest):
test_git_uris = {
# version pattern "X.Y.Z"
- ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+ ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
: "1.99.4",
# version pattern "vX.Y"
# mirror of git.infradead.org since network issues interfered with testing
@@ -1258,9 +1262,9 @@ class FetchLatestVersionTest(FetcherTest):
: "0.4.3",
("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
: "11.0.0",
- ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+ ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
: "1.3.59",
- ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+ ("remake", "git://github.com/rocky/remake.git;protocol=https", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
: "3.82+dbg0.9",
}
@@ -1354,9 +1358,6 @@ class FetchCheckStatusTest(FetcherTest):
"http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
"http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
"ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
- "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
- "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
- "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
# GitHub releases are hosted on Amazon S3, which doesn't support HEAD
"https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
]
@@ -2047,7 +2048,7 @@ class GitShallowTest(FetcherTest):
@skipIfNoNetwork()
def test_bitbake(self):
- self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
+ self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
self.git('config core.bare true', cwd=self.srcdir)
self.git('fetch', cwd=self.srcdir)
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index b282d09abfce..2a150fe9c73e 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -16,7 +16,8 @@ import bb.msg
import multiprocessing
import fcntl
import importlib
-from importlib import machinery
+import importlib.machinery
+import importlib.util
import itertools
import subprocess
import glob
@@ -451,6 +452,10 @@ def lockfile(name, shared=False, retry=True, block=False):
consider the possibility of sending a signal to the process to break
out - at which point you want block=True rather than retry=True.
"""
+ if len(name) > 255:
+ root, ext = os.path.splitext(name)
+ name = root[:255 - len(ext)] + ext
+
dirname = os.path.dirname(name)
mkdirhier(dirname)
@@ -487,7 +492,7 @@ def lockfile(name, shared=False, retry=True, block=False):
return lf
lf.close()
except OSError as e:
- if e.errno == errno.EACCES:
+ if e.errno == errno.EACCES or e.errno == errno.ENAMETOOLONG:
logger.error("Unable to acquire lock '%s', %s",
e.strerror, name)
sys.exit(1)
@@ -1616,7 +1621,9 @@ def load_plugins(logger, plugins, pluginpath):
logger.debug('Loading plugin %s' % name)
spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
if spec:
- return spec.loader.load_module()
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
logger.debug('Loading plugins from %s...' % pluginpath)
diff --git a/bitbake/lib/hashserv/server.py b/bitbake/lib/hashserv/server.py
index a0dc0c170f2b..df0fa0a07937 100644
--- a/bitbake/lib/hashserv/server.py
+++ b/bitbake/lib/hashserv/server.py
@@ -521,7 +521,7 @@ class Server(object):
def start_tcp_server(self, host, port):
self.server = self.loop.run_until_complete(
- asyncio.start_server(self.handle_client, host, port, loop=self.loop)
+ asyncio.start_server(self.handle_client, host, port)
)
for s in self.server.sockets:
@@ -546,7 +546,7 @@ class Server(object):
# Work around path length limits in AF_UNIX
os.chdir(os.path.dirname(path))
self.server = self.loop.run_until_complete(
- asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+ asyncio.start_unix_server(self.handle_client, os.path.basename(path))
)
finally:
os.chdir(cwd)
diff --git a/bitbake/lib/toaster/tests/builds/buildtest.py b/bitbake/lib/toaster/tests/builds/buildtest.py
index 872bbd377510..13b51fb0d8e4 100644
--- a/bitbake/lib/toaster/tests/builds/buildtest.py
+++ b/bitbake/lib/toaster/tests/builds/buildtest.py
@@ -119,7 +119,7 @@ class BuildTest(unittest.TestCase):
if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
ProjectVariable.objects.get_or_create(
name="SSTATE_MIRRORS",
- value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH",
+ value="file://.* http://sstate.yoctoproject.org/PATH;downloadfilename=PATH",
project=project)
ProjectTarget.objects.create(project=project,
--
2.35.1
^ permalink raw reply [flat|nested] 7+ messages in thread* [PATCH v3 3/3] wic: Update to the latest revision
2022-04-22 9:05 [PATCH v3 0/3] bump bitbake and wic for python 3.10 support Henning Schild
2022-04-22 9:06 ` [PATCH v3 1/3] bitbake: Update to 1.50.5 release Henning Schild
2022-04-22 9:06 ` [PATCH v3 2/3] wic: align our fork of common.wks.inc with wic version we carry Henning Schild
@ 2022-04-22 9:06 ` Henning Schild
2022-04-22 9:57 ` [PATCH v3 0/3] bump bitbake and wic for python 3.10 support Anton Mikanovich
2022-04-23 5:08 ` Anton Mikanovich
4 siblings, 0 replies; 7+ messages in thread
From: Henning Schild @ 2022-04-22 9:06 UTC (permalink / raw)
To: isar-users; +Cc: Florian Bezdeka, Felix Moessbauer, Henning Schild
Update to the latest wic from OE-core.
OE-core Revision: 712552b5cc427d7be1258c45886de9b57f7272c9
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
.../wic/plugins/source/bootimg-efi-isar.py | 77 ++++++++++++++++---
.../wic/plugins/source/bootimg-pcbios-isar.py | 6 +-
scripts/lib/wic/canned-wks/common.wks.inc | 2 +-
scripts/lib/wic/canned-wks/directdisk-gpt.wks | 2 +-
scripts/lib/wic/canned-wks/mkefidisk.wks | 2 +-
scripts/lib/wic/engine.py | 6 +-
scripts/lib/wic/help.py | 10 ++-
scripts/lib/wic/ksparser.py | 8 +-
scripts/lib/wic/misc.py | 4 +-
scripts/lib/wic/partition.py | 25 ++++--
scripts/lib/wic/pluginbase.py | 8 +-
scripts/lib/wic/plugins/imager/direct.py | 11 ++-
scripts/lib/wic/plugins/source/bootimg-efi.py | 74 +++++++++++++++---
.../lib/wic/plugins/source/bootimg-pcbios.py | 6 +-
scripts/lib/wic/plugins/source/rawcopy.py | 35 ++++++++-
scripts/lib/wic/plugins/source/rootfs.py | 2 +-
scripts/wic | 9 ++-
17 files changed, 231 insertions(+), 56 deletions(-)
diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py
index 5ba0777ad244..a24e04f309da 100644
--- a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py
+++ b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py
@@ -12,6 +12,7 @@
import logging
import os
+import tempfile
import shutil
import re
@@ -129,12 +130,13 @@ class BootimgEFIPlugin(SourcePlugin):
bootloader = creator.ks.bootloader
loader_conf = ""
- loader_conf += "default boot\n"
+ if source_params.get('create-unified-kernel-image') != "true":
+ loader_conf += "default boot\n"
loader_conf += "timeout %d\n" % bootloader.timeout
initrd = source_params.get('initrd')
- if initrd:
+ if initrd and source_params.get('create-unified-kernel-image') != "true":
# obviously we need to have a common common deploy var
bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
if not bootimg_dir:
@@ -195,11 +197,12 @@ class BootimgEFIPlugin(SourcePlugin):
for rd in initrds:
boot_conf += "initrd /%s\n" % rd
- logger.debug("Writing systemd-boot config "
- "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
- cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
- cfg.write(boot_conf)
- cfg.close()
+ if source_params.get('create-unified-kernel-image') != "true":
+ logger.debug("Writing systemd-boot config "
+ "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
+ cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
+ cfg.write(boot_conf)
+ cfg.close()
@classmethod
@@ -300,11 +303,63 @@ class BootimgEFIPlugin(SourcePlugin):
kernel = "%s-%s.bin" % \
(get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
- install_cmd = "install -m 0644 %s/%s %s/%s" % \
- (staging_kernel_dir, kernel, hdddir, kernel)
+ if source_params.get('create-unified-kernel-image') == "true":
+ initrd = source_params.get('initrd')
+ if not initrd:
+ raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting")
+
+ deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub"))
+ if len(efi_stub) == 0:
+ raise WicError("Unified Kernel Image EFI stub not found, exiting")
+ efi_stub = efi_stub[0]
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ label = source_params.get('label')
+ label_conf = "root=%s" % creator.rootdev
+ if label:
+ label_conf = "LABEL=%s" % label
+
+ bootloader = creator.ks.bootloader
+ cmdline = open("%s/cmdline" % tmp_dir, "w")
+ cmdline.write("%s %s" % (label_conf, bootloader.append))
+ cmdline.close()
+
+ initrds = initrd.split(';')
+ initrd = open("%s/initrd" % tmp_dir, "wb")
+ for f in initrds:
+ with open("%s/%s" % (deploy_dir, f), 'rb') as in_file:
+ shutil.copyfileobj(in_file, initrd)
+ initrd.close()
+
+ # Searched by systemd-boot:
+ # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images
+ install_cmd = "install -d %s/EFI/Linux" % hdddir
+ exec_cmd(install_cmd)
+
+ staging_dir_host = get_bitbake_var("STAGING_DIR_HOST")
+
+ # https://www.freedesktop.org/software/systemd/man/systemd-stub.html
+ objcopy_cmd = "objcopy \
+ --add-section .osrel=%s --change-section-vma .osrel=0x20000 \
+ --add-section .cmdline=%s --change-section-vma .cmdline=0x30000 \
+ --add-section .linux=%s --change-section-vma .linux=0x2000000 \
+ --add-section .initrd=%s --change-section-vma .initrd=0x3000000 \
+ %s %s" % \
+ ("%s/usr/lib/os-release" % staging_dir_host,
+ cmdline.name,
+ "%s/%s" % (staging_kernel_dir, kernel),
+ initrd.name,
+ efi_stub,
+ "%s/EFI/Linux/linux.efi" % hdddir)
+ exec_cmd(objcopy_cmd)
+ else:
+ install_cmd = "install -m 0644 %s/%s %s/%s" % \
+ (staging_kernel_dir, kernel, hdddir, kernel)
+
+ install_cmd = isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir)
+ exec_cmd(install_cmd)
- install_cmd = isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir)
- exec_cmd(install_cmd)
if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
for src_path, dst_path in cls.install_task:
diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py
index 9136d4f215ca..7fb0f2a87394 100644
--- a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py
+++ b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py
@@ -213,8 +213,10 @@ class BootimgPcbiosIsarPlugin(SourcePlugin):
# dosfs image, created by mkdosfs
bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
- dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \
- (part.fsuuid, bootimg, blocks)
+ label = part.label if part.label else "boot"
+
+ dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
+ (label, part.fsuuid, bootimg, blocks)
exec_native_cmd(dosfs_cmd, native_sysroot)
mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc
index 4fd29fa8c119..89880b417b6e 100644
--- a/scripts/lib/wic/canned-wks/common.wks.inc
+++ b/scripts/lib/wic/canned-wks/common.wks.inc
@@ -1,3 +1,3 @@
# This file is included into 3 canned wks files from this directory
part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --use-uuid --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024
+part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024
diff --git a/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
index cf16c0c30bbd..8d7d8de6ea7e 100644
--- a/scripts/lib/wic/canned-wks/directdisk-gpt.wks
+++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
@@ -4,7 +4,7 @@
part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
diff --git a/scripts/lib/wic/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks
index d1878e23e5a3..9f534fe18471 100644
--- a/scripts/lib/wic/canned-wks/mkefidisk.wks
+++ b/scripts/lib/wic/canned-wks/mkefidisk.wks
@@ -4,7 +4,7 @@
part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
part swap --ondisk sda --size 44 --label swap1 --fstype=swap
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 018815b96688..674ccfc24418 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -19,10 +19,10 @@ import os
import tempfile
import json
import subprocess
+import shutil
import re
from collections import namedtuple, OrderedDict
-from distutils.spawn import find_executable
from wic import WicError
from wic.filemap import sparse_copy
@@ -245,7 +245,7 @@ class Disk:
for path in pathlist.split(':'):
self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
- self.parted = find_executable("parted", self.paths)
+ self.parted = shutil.which("parted", path=self.paths)
if not self.parted:
raise WicError("Can't find executable parted")
@@ -283,7 +283,7 @@ class Disk:
"resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
aname = "_%s" % name
if aname not in self.__dict__:
- setattr(self, aname, find_executable(name, self.paths))
+ setattr(self, aname, shutil.which(name, path=self.paths))
if aname not in self.__dict__ or self.__dict__[aname] is None:
raise WicError("Can't find executable '{}'".format(name))
return self.__dict__[aname]
diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py
index bd3a2b97dfaf..4ff7470a6a79 100644
--- a/scripts/lib/wic/help.py
+++ b/scripts/lib/wic/help.py
@@ -637,7 +637,7 @@ DESCRIPTION
oe-core: directdisk.bbclass and mkefidisk.sh. The difference
between wic and those examples is that with wic the functionality
of those scripts is implemented by a general-purpose partitioning
- 'language' based on Redhat kickstart syntax).
+ 'language' based on Red Hat kickstart syntax).
The initial motivation and design considerations that lead to the
current tool are described exhaustively in Yocto Bug #3847
@@ -840,8 +840,8 @@ DESCRIPTION
meanings. The commands are based on the Fedora kickstart
documentation but with modifications to reflect wic capabilities.
- http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition
- http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader
+ https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition
+ https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader
Commands
@@ -930,6 +930,7 @@ DESCRIPTION
ext4
btrfs
squashfs
+ erofs
swap
--fsoptions: Specifies a free-form string of options to be
@@ -990,6 +991,9 @@ DESCRIPTION
multiple partitions and we want to keep the right
permissions and usernames in all the partitions.
+ --no-fstab-update: This option is specific to wic. It does not update the
+ '/etc/fstab' stock file for the given partition.
+
--extra-space: This option is specific to wic. It adds extra
space after the space filled by the content
of the partition. The final size can go
diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py
index 3eb669da39ca..0df9eb0d057d 100644
--- a/scripts/lib/wic/ksparser.py
+++ b/scripts/lib/wic/ksparser.py
@@ -157,7 +157,8 @@ class KickStart():
part.add_argument('--fsoptions', dest='fsopts')
part.add_argument('--fstype', default='vfat',
choices=('ext2', 'ext3', 'ext4', 'btrfs',
- 'squashfs', 'vfat', 'msdos', 'swap'))
+ 'squashfs', 'vfat', 'msdos', 'erofs',
+ 'swap'))
part.add_argument('--mkfs-extraopts', default='')
part.add_argument('--label')
part.add_argument('--use-label', action='store_true')
@@ -184,6 +185,7 @@ class KickStart():
part.add_argument('--use-uuid', action='store_true')
part.add_argument('--uuid')
part.add_argument('--fsuuid')
+ part.add_argument('--no-fstab-update', action='store_true')
bootloader = subparsers.add_parser('bootloader')
bootloader.add_argument('--append')
@@ -229,6 +231,10 @@ class KickStart():
err = "%s:%d: SquashFS does not support LABEL" \
% (confpath, lineno)
raise KickStartError(err)
+ # erofs does not support filesystem labels
+ if parsed.fstype == 'erofs' and parsed.label:
+ err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+ raise KickStartError(err)
if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
if parsed.fsuuid:
if parsed.fsuuid.upper().startswith('0X'):
diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py
index 57c042c503e6..3e118229960b 100644
--- a/scripts/lib/wic/misc.py
+++ b/scripts/lib/wic/misc.py
@@ -16,9 +16,9 @@ import logging
import os
import re
import subprocess
+import shutil
from collections import defaultdict
-from distutils import spawn
from wic import WicError
@@ -122,7 +122,7 @@ def find_executable(cmd, paths):
if provided and "%s-native" % recipe in provided:
return True
- return spawn.find_executable(cmd, paths)
+ return shutil.which(cmd, path=paths)
def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
"""
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index 76d144d12d30..09e491dd494d 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -54,6 +54,7 @@ class Partition():
self.uuid = args.uuid
self.fsuuid = args.fsuuid
self.type = args.type
+ self.no_fstab_update = args.no_fstab_update
self.updated_fstab_path = None
self.has_fstab = False
self.update_fstab_in_rootfs = False
@@ -104,7 +105,7 @@ class Partition():
extra_blocks = self.extra_space
rootfs_size = actual_rootfs_size + extra_blocks
- rootfs_size *= self.overhead_factor
+ rootfs_size = int(rootfs_size * self.overhead_factor)
logger.debug("Added %d extra blocks to %s to get to %d total blocks",
extra_blocks, self.mountpoint, rootfs_size)
@@ -141,9 +142,9 @@ class Partition():
native_sysroot)
self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
else:
- if self.fstype == 'squashfs':
- raise WicError("It's not possible to create empty squashfs "
- "partition '%s'" % (self.mountpoint))
+ if self.fstype in ('squashfs', 'erofs'):
+ raise WicError("It's not possible to create empty %s "
+ "partition '%s'" % (self.fstype, self.mountpoint))
rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
self.lineno, self.fstype)
@@ -170,7 +171,7 @@ class Partition():
# Split sourceparams string of the form key1=val1[,key2=val2,...]
# into a dict. Also accepts valueless keys i.e. without =
splitted = self.sourceparams.split(',')
- srcparams_dict = dict(par.split('=', 1) for par in splitted if par)
+ srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par)
plugin = PluginMgr.get_plugins('source')[self.source]
plugin.do_configure_partition(self, srcparams_dict, creator,
@@ -286,7 +287,7 @@ class Partition():
(self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
- if self.updated_fstab_path and self.has_fstab:
+ if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
with open(debugfs_script_path, "w") as f:
f.write("cd etc\n")
@@ -350,7 +351,7 @@ class Partition():
mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
exec_native_cmd(mcopy_cmd, native_sysroot)
- if self.updated_fstab_path and self.has_fstab:
+ if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
exec_native_cmd(mcopy_cmd, native_sysroot)
@@ -369,6 +370,16 @@ class Partition():
(rootfs_dir, rootfs, extraopts)
exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
+ def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+ native_sysroot, pseudo):
+ """
+ Prepare content for a erofs rootfs partition.
+ """
+ extraopts = self.mkfs_extraopts or ''
+ erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+ (extraopts, self.fsuuid, rootfs, rootfs_dir)
+ exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+
def prepare_empty_partition_ext(self, rootfs, oe_builddir,
native_sysroot):
"""
diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py
index d9b4e57747e7..b64568339b1b 100644
--- a/scripts/lib/wic/pluginbase.py
+++ b/scripts/lib/wic/pluginbase.py
@@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin']
import os
import logging
+import types
from collections import defaultdict
-from importlib.machinery import SourceFileLoader
+import importlib
+import importlib.util
from wic import WicError
from wic.misc import get_bitbake_var
@@ -54,7 +56,9 @@ class PluginMgr:
mname = fname[:-3]
mpath = os.path.join(ppath, fname)
logger.debug("loading plugin module %s", mpath)
- SourceFileLoader(mname, mpath).load_module()
+ spec = importlib.util.spec_from_file_location(mname, mpath)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
return PLUGINS.get(ptype)
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index ea709e8c545d..35fff7c10242 100644
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -77,7 +77,8 @@ class DirectPlugin(ImagerPlugin):
image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
self._image = PartitionedImage(image_path, self.ptable_format,
- self.parts, self.native_sysroot)
+ self.parts, self.native_sysroot,
+ options.extra_space)
def setup_workdir(self, workdir):
if workdir:
@@ -116,7 +117,7 @@ class DirectPlugin(ImagerPlugin):
updated = False
for part in self.parts:
if not part.realnum or not part.mountpoint \
- or part.mountpoint == "/":
+ or part.mountpoint == "/" or not part.mountpoint.startswith('/'):
continue
if part.use_uuid:
@@ -258,6 +259,8 @@ class DirectPlugin(ImagerPlugin):
if part.mountpoint == "/":
if part.uuid:
return "PARTUUID=%s" % part.uuid
+ elif part.label:
+ return "PARTLABEL=%s" % part.label
else:
suffix = 'p' if part.disk.startswith('mmcblk') else ''
return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
@@ -293,7 +296,7 @@ class PartitionedImage():
Partitioned image in a file.
"""
- def __init__(self, path, ptable_format, partitions, native_sysroot=None):
+ def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0):
self.path = path # Path to the image file
self.numpart = 0 # Number of allocated partitions
self.realpart = 0 # Number of partitions in the partition table
@@ -314,6 +317,7 @@ class PartitionedImage():
self.sector_size = SECTOR_SIZE
self.native_sysroot = native_sysroot
num_real_partitions = len([p for p in self.partitions if not p.no_table])
+ self.extra_space = extra_space
# calculate the real partition number, accounting for partitions not
# in the partition table and logical partitions
@@ -483,6 +487,7 @@ class PartitionedImage():
self.min_size += GPT_OVERHEAD
self.min_size *= self.sector_size
+ self.min_size += self.extra_space
def _create_partition(self, device, parttype, fstype, start, size):
""" Create a partition on an image described by the 'device' object. """
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py
index cdc72543c200..0391aebdc840 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -12,6 +12,7 @@
import logging
import os
+import tempfile
import shutil
import re
@@ -119,12 +120,13 @@ class BootimgEFIPlugin(SourcePlugin):
bootloader = creator.ks.bootloader
loader_conf = ""
- loader_conf += "default boot\n"
+ if source_params.get('create-unified-kernel-image') != "true":
+ loader_conf += "default boot\n"
loader_conf += "timeout %d\n" % bootloader.timeout
initrd = source_params.get('initrd')
- if initrd:
+ if initrd and source_params.get('create-unified-kernel-image') != "true":
# obviously we need to have a common common deploy var
bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
if not bootimg_dir:
@@ -183,11 +185,12 @@ class BootimgEFIPlugin(SourcePlugin):
for rd in initrds:
boot_conf += "initrd /%s\n" % rd
- logger.debug("Writing systemd-boot config "
- "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
- cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
- cfg.write(boot_conf)
- cfg.close()
+ if source_params.get('create-unified-kernel-image') != "true":
+ logger.debug("Writing systemd-boot config "
+ "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
+ cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
+ cfg.write(boot_conf)
+ cfg.close()
@classmethod
@@ -288,9 +291,60 @@ class BootimgEFIPlugin(SourcePlugin):
kernel = "%s-%s.bin" % \
(get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
- install_cmd = "install -m 0644 %s/%s %s/%s" % \
- (staging_kernel_dir, kernel, hdddir, kernel)
- exec_cmd(install_cmd)
+ if source_params.get('create-unified-kernel-image') == "true":
+ initrd = source_params.get('initrd')
+ if not initrd:
+ raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting")
+
+ deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub"))
+ if len(efi_stub) == 0:
+ raise WicError("Unified Kernel Image EFI stub not found, exiting")
+ efi_stub = efi_stub[0]
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ label = source_params.get('label')
+ label_conf = "root=%s" % creator.rootdev
+ if label:
+ label_conf = "LABEL=%s" % label
+
+ bootloader = creator.ks.bootloader
+ cmdline = open("%s/cmdline" % tmp_dir, "w")
+ cmdline.write("%s %s" % (label_conf, bootloader.append))
+ cmdline.close()
+
+ initrds = initrd.split(';')
+ initrd = open("%s/initrd" % tmp_dir, "wb")
+ for f in initrds:
+ with open("%s/%s" % (deploy_dir, f), 'rb') as in_file:
+ shutil.copyfileobj(in_file, initrd)
+ initrd.close()
+
+ # Searched by systemd-boot:
+ # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images
+ install_cmd = "install -d %s/EFI/Linux" % hdddir
+ exec_cmd(install_cmd)
+
+ staging_dir_host = get_bitbake_var("STAGING_DIR_HOST")
+
+ # https://www.freedesktop.org/software/systemd/man/systemd-stub.html
+ objcopy_cmd = "objcopy \
+ --add-section .osrel=%s --change-section-vma .osrel=0x20000 \
+ --add-section .cmdline=%s --change-section-vma .cmdline=0x30000 \
+ --add-section .linux=%s --change-section-vma .linux=0x2000000 \
+ --add-section .initrd=%s --change-section-vma .initrd=0x3000000 \
+ %s %s" % \
+ ("%s/usr/lib/os-release" % staging_dir_host,
+ cmdline.name,
+ "%s/%s" % (staging_kernel_dir, kernel),
+ initrd.name,
+ efi_stub,
+ "%s/EFI/Linux/linux.efi" % hdddir)
+ exec_cmd(objcopy_cmd)
+ else:
+ install_cmd = "install -m 0644 %s/%s %s/%s" % \
+ (staging_kernel_dir, kernel, hdddir, kernel)
+ exec_cmd(install_cmd)
if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
for src_path, dst_path in cls.install_task:
diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
index f2639e700493..32e47f183146 100644
--- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
+++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
@@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin):
# dosfs image, created by mkdosfs
bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
- dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \
- (part.fsuuid, bootimg, blocks)
+ label = part.label if part.label else "boot"
+
+ dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
+ (label, part.fsuuid, bootimg, blocks)
exec_native_cmd(dosfs_cmd, native_sysroot)
mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py
index 3c4997d8ba5e..7c90cd3cf82b 100644
--- a/scripts/lib/wic/plugins/source/rawcopy.py
+++ b/scripts/lib/wic/plugins/source/rawcopy.py
@@ -4,6 +4,8 @@
import logging
import os
+import signal
+import subprocess
from wic import WicError
from wic.pluginbase import SourcePlugin
@@ -29,15 +31,34 @@ class RawCopyPlugin(SourcePlugin):
cmd = 'btrfs filesystem label %s %s' % (dst, label)
elif fstype == 'swap':
cmd = 'mkswap -L %s %s' % (label, dst)
- elif fstype == 'squashfs':
- raise WicError("It's not possible to update a squashfs "
- "filesystem label '%s'" % (label))
+ elif fstype in ('squashfs', 'erofs'):
+ raise WicError("It's not possible to update a %s "
+ "filesystem label '%s'" % (fstype, label))
else:
raise WicError("Cannot update filesystem label: "
"Unknown fstype: '%s'" % (fstype))
exec_cmd(cmd)
+ @staticmethod
+ def do_image_uncompression(src, dst, workdir):
+ def subprocess_setup():
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ # SIGPIPE errors are known issues with gzip/bash
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+ extension = os.path.splitext(src)[1]
+ decompressor = {
+ ".bz2": "bzip2",
+ ".gz": "gzip",
+ ".xz": "xz"
+ }.get(extension)
+ if not decompressor:
+ raise WicError("Not supported compressor filename extension: %s" % extension)
+ cmd = "%s -dc %s > %s" % (decompressor, src, dst)
+ subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir)
+
@classmethod
def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
@@ -56,7 +77,13 @@ class RawCopyPlugin(SourcePlugin):
if 'file' not in source_params:
raise WicError("No file specified")
- src = os.path.join(kernel_dir, source_params['file'])
+ if 'unpack' in source_params:
+ img = os.path.join(kernel_dir, source_params['file'])
+ src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0])
+ RawCopyPlugin.do_image_uncompression(img, src, cr_workdir)
+ else:
+ src = os.path.join(kernel_dir, source_params['file'])
+
dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
if not os.path.exists(os.path.dirname(dst)):
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index 96d940a91d61..2e34e715ca73 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -218,7 +218,7 @@ class RootfsPlugin(SourcePlugin):
# Update part.has_fstab here as fstab may have been added or
# removed by the above modifications.
part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
- if part.update_fstab_in_rootfs and part.has_fstab:
+ if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
fstab_path = os.path.join(new_rootfs, "etc/fstab")
# Assume that fstab should always be owned by root with fixed permissions
install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path)
diff --git a/scripts/wic b/scripts/wic
index a741aed364bf..aee63a45aacf 100755
--- a/scripts/wic
+++ b/scripts/wic
@@ -22,9 +22,9 @@ import sys
import argparse
import logging
import subprocess
+import shutil
from collections import namedtuple
-from distutils import spawn
# External modules
scripts_path = os.path.dirname(os.path.realpath(__file__))
@@ -47,7 +47,7 @@ if os.environ.get('SDKTARGETSYSROOT'):
break
sdkroot = os.path.dirname(sdkroot)
-bitbake_exe = spawn.find_executable('bitbake')
+bitbake_exe = shutil.which('bitbake')
if bitbake_exe:
bitbake_path = scriptpath.add_bitbake_lib_path()
import bb
@@ -159,6 +159,9 @@ def wic_create_subcommand(options, usage_str):
"(Use -e/--image-name to specify it)")
native_sysroot = options.native_sysroot
+ if options.kernel_dir:
+ kernel_dir = options.kernel_dir
+
if not options.vars_dir and (not native_sysroot or not os.path.isdir(native_sysroot)):
logger.info("Building wic-tools...\n")
subprocess.check_call(["bitbake", "wic-tools"])
@@ -346,6 +349,8 @@ def wic_init_parser_create(subparser):
default=False, help="output debug information")
subparser.add_argument("-i", "--imager", dest="imager",
default="direct", help="the wic imager plugin")
+ subparser.add_argument("--extra-space", type=int, dest="extra_space",
+ default=0, help="additional free disk space to add to the image")
return
--
2.35.1
^ permalink raw reply [flat|nested] 7+ messages in thread