* [PATCH v4 01/13] scripts: Add debrepo python script handling base-apt
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 02/13] meta: Add debrepo bbclass handling base-apt prefetching Uladzimir Bely
` (11 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
This is the main utility responsible for prefetching packages
into local `base-apt` repo from external Debian mirrors. It uses
python-apt module and requires some kind of minimal `rootfs` to work
(let's call it "debrepo context").
Once initialized with `--init --workdir=<path>`, it stores the initial
configuration in `repo.opts` file inside the context and uses it at
futher calls.
In future, the logic `debrepo` script implements could be directly
implemented inside bitbake classes.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
scripts/debrepo | 440 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 440 insertions(+)
create mode 100755 scripts/debrepo
diff --git a/scripts/debrepo b/scripts/debrepo
new file mode 100755
index 00000000..d3965f14
--- /dev/null
+++ b/scripts/debrepo
@@ -0,0 +1,440 @@
+#!/usr/bin/env python3
+
+# This software is a part of ISAR.
+# Copyright (C) 2022 ilbers GmbH
+
+import os
+import sys
+import fcntl
+
+import shutil
+import subprocess
+import getopt
+import pickle
+import urllib.parse
+
+import apt_pkg
+import apt.progress.base
+
+
+class DebRepo(object):
+ def __init__(self, workdir, cmdline_opts):
+ self.workdir = workdir
+ self.optsfile = self.workdir + "/repo.opts"
+
+ # Set default values
+ self.distro = "debian"
+
+ self.repo = self.workdir + "/repo/apt" + "/" + self.distro
+ self.repodb = self.workdir + "/repo/db" + "/" + self.distro
+ self.mirror = "http://deb.debian.org/debian"
+ self.arch = "amd64"
+ self.codename = "bullseye"
+ self.keydir = "/etc/apt/trusted.gpg.d"
+ self.check_gpg = True
+ self.isaraptdir = ""
+
+ # Load stored opts
+ opts = self.load_opts()
+ print("stored opts: " + str(opts))
+
+ # Overwrite opts by cmdline_opts
+ for opt, arg in cmdline_opts.items():
+ opts[opt] = arg
+
+ print("all opts: " + str(opts))
+
+ # Replace by values passed in commandline
+ for opt, arg in opts.items():
+ if opt == "mirror":
+ self.mirror = arg
+ if opt == "arch":
+ self.arch = arg
+ if opt == "distro":
+ self.distro = arg
+ if opt == "codename":
+ self.codename = arg
+ if opt == "keydir":
+ self.keydir = arg
+ if opt == "isaraptdir":
+ self.isaraptdir = arg
+ if opt == "check_gpg":
+ self.check_gpg = arg
+
+ self.crossarch = self.arch
+ for opt, arg in opts.items():
+ if opt == "crossarch":
+ self.crossarch = arg
+
+ self.compatarch = ""
+ for opt, arg in opts.items():
+ if opt == "compatarch":
+ self.compatarch = arg
+
+ for opt, arg in opts.items():
+ if opt == "repodir":
+ self.repo = arg + "/" + self.distro
+ if opt == "repodbdir":
+ self.repodb = arg + "/" + self.distro
+
+ self.save_opts(opts)
+
+ print("workdir: " + str(self.workdir))
+ print("repo: " + str(self.repo))
+ print("repodb: " + str(self.repodb))
+ print("mirror: " + str(self.mirror))
+ print("arch: " + str(self.arch))
+ print("crossarch: " + str(self.crossarch))
+ if self.compatarch:
+ print("compatarch: " + str(self.compatarch))
+ print("distro: " + str(self.distro))
+ print("codename: " + str(self.codename))
+ print("keydir: " + str(self.keydir))
+ print("isaraptdir: " + str(self.isaraptdir))
+ print("check_gpg: " + str(self.check_gpg))
+
+ self.cache = None
+ self.depcache = None
+ self.sr = None
+
+ def create_rootfs(self, aptsrcsfile):
+ if not os.path.exists(self.workdir + "/var/lib/dpkg"):
+ os.makedirs(self.workdir + "/var/lib/dpkg")
+ with open(self.workdir + "/var/lib/dpkg" + "/status", "w"):
+ pass
+
+ if not os.path.exists(self.workdir + "/etc/apt/sources.list.d"):
+ os.makedirs(self.workdir + "/etc/apt/sources.list.d")
+ if os.path.exists(aptsrcsfile):
+ shutil.copy(aptsrcsfile, self.workdir + "/etc/apt/sources.list.d/bootstrap.list")
+
+ if self.isaraptdir:
+ with open(self.workdir + "/etc/apt/sources.list.d/isar-apt.list", "w") as f:
+ f.write("deb [trusted=yes] file://" + self.isaraptdir + " isar main\n")
+
+ if not os.path.exists(self.workdir + "/var/cache/apt/archives/partial"):
+ os.makedirs(self.workdir + "/var/cache/apt/archives/partial")
+
+ if not os.path.exists(self.workdir + "/tmp"):
+ os.makedirs(self.workdir + "/tmp")
+
+ def apt_config(self, init, crossbuild):
+ # Configure apt to work with empty directory
+ if not init and self.arch != self.crossarch:
+ apt_pkg.config["APT::Architectures::"] = self.crossarch
+ apt_pkg.config["APT::Architectures::"] = self.arch
+
+ if not init and self.compatarch:
+ apt_pkg.config["APT::Architectures::"] = self.compatarch
+
+ apt_pkg.config.set("APT::Architecture", self.arch)
+
+ apt_pkg.config.set("Dir", self.workdir)
+ apt_pkg.config.set("Dir::Cache", self.workdir + "/var/cache/apt")
+ apt_pkg.config.set("Dir::State::status", self.workdir + "/var/lib/dpkg/status")
+
+ apt_pkg.config.set("APT::Install-Recommends", "0")
+ apt_pkg.config.set("APT::Install-Suggests", "0")
+
+ # Use host keys for authentification
+ # apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
+ # apt_pkg.config.set("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d")
+ apt_pkg.config.set("Dir::Etc::TrustedParts", self.keydir)
+
+ # Allow using repositories without keys
+ if not self.check_gpg:
+ apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
+
+ def mark_essential(self):
+ for pkg in self.cache.packages:
+ if pkg.architecture == self.arch:
+ if pkg.essential:
+ self.depcache.mark_install(pkg)
+
+ def mark_by_prio(self, priority):
+ for pkg in self.cache.packages:
+ if pkg.architecture == self.arch:
+ ver = self.depcache.get_candidate_ver(pkg)
+ if ver and ver.priority <= priority:
+ self.depcache.mark_install(pkg)
+
+ def mark_pkg(self, pkgname, crossbuild):
+ if pkgname in self.cache:
+ pkg = self.cache[pkgname]
+
+ if not crossbuild or ':' in pkgname or len(pkg.version_list) == 0:
+ if pkg.has_provides and not pkg.has_versions:
+ print("pkgname is virtual package, selecting best provide")
+ # Select first provide
+ pkg_provide = pkg.provides_list[0][2]
+ # Find better provide with higher version
+ for provide in pkg.provides_list:
+ if apt_pkg.version_compare(provide[2].ver_str, pkg_provide.ver_str) > 0:
+ pkg_provide = provide[2]
+ self.depcache.mark_install(pkg_provide.parent_pkg)
+ else:
+ self.depcache.mark_install(pkg)
+ else:
+ version = pkg.version_list[0]
+ if version.arch == "all":
+ self.depcache.mark_install(pkg)
+ else:
+ if version.multi_arch == version.MULTI_ARCH_FOREIGN:
+ if (pkgname, self.arch) in self.cache:
+ nativepkg = self.cache[pkgname, self.arch]
+ self.depcache.mark_install(nativepkg)
+ else:
+ if (pkgname, self.crossarch) in self.cache:
+ crosspkg = self.cache[pkgname, self.crossarch]
+ self.depcache.mark_install(crosspkg)
+
+ def mark_list(self, pkglist, crossbuild):
+ if pkglist:
+ for pkgname in pkglist:
+ self.mark_pkg(pkgname, crossbuild)
+
+ def handle_deb(self, item):
+ lockfd = open(self.repo + '/../repo.lock', 'w')
+ fcntl.flock(lockfd,fcntl.LOCK_EX)
+ subprocess.run([
+ "reprepro",
+ "--dbdir", self.repodb,
+ "--outdir", self.repo,
+ "--confdir", self.repo + "/conf",
+ "-C", "main",
+ "includedeb",
+ self.codename,
+ item.destfile
+ ])
+ lockfd.close()
+
+ def handle_repo(self, fetcher):
+ fetcher.run()
+ for item in fetcher.items:
+ if item.status == item.STAT_ERROR:
+ print("Some error ocured: '%s'" % item.error_text)
+ pass
+ else:
+ self.handle_deb(item)
+
+ def get_filename(self, uri):
+ path = urllib.parse.urlparse(uri).path
+ unquoted_path = urllib.parse.unquote(path)
+ basename = os.path.basename(unquoted_path)
+ return basename
+
+ def download_file(self, uri):
+ filename = self.get_filename(uri)
+ subprocess.run([
+ "wget",
+ "-H",
+ "--timeout=30",
+ "--tries=3",
+ "-q",
+ uri,
+ "-O",
+ self.workdir + "/tmp/" + filename
+ ])
+
+ def handle_dsc(self, uri):
+ filename = self.get_filename(uri)
+ lockfd = open(self.repo + '/../repo.lock', 'w')
+ fcntl.flock(lockfd,fcntl.LOCK_EX)
+ subprocess.run([
+ "reprepro",
+ "--dbdir", self.repodb,
+ "--outdir", self.repo,
+ "--confdir", self.repo + "/conf",
+ "-C", "main",
+ "-S", "-", "-P" "source",
+ "--delete",
+ "includedsc",
+ self.codename,
+ os.path.realpath(self.workdir + "/tmp/" + filename)
+ ])
+ lockfd.close()
+
+ def handle_src_list(self, pkgs):
+ if pkgs:
+ for pkg in pkgs:
+ pkgname=pkg
+ pkgver=""
+ if '=' in pkg:
+ pkgname = pkg.split("=")[0]
+ pkgver = pkg.split("=")[1]
+
+ self.sr.restart()
+ while self.sr.lookup(pkgname):
+ if pkgver and pkgver != self.sr.version:
+ continue
+
+ for sr_file in self.sr.files:
+ print(self.sr.index.archive_uri(sr_file[2]))
+ self.download_file(self.sr.index.archive_uri(sr_file[2]))
+
+ dsc_uri = self.sr.index.archive_uri(self.sr.files[0][2])
+ self.handle_dsc(dsc_uri)
+ break
+
+ def apt_run(self, init, srcmode, pkgs, controlfile, crossbuild):
+ apt_pkg.init()
+
+ sources = apt_pkg.SourceList()
+ sources.read_main_list()
+
+ progress = apt.progress.text.AcquireProgress()
+
+ self.cache = apt_pkg.Cache()
+ self.cache.update(progress, sources)
+ self.cache = apt_pkg.Cache()
+
+ self.depcache = apt_pkg.DepCache(self.cache)
+ self.sr = apt_pkg.SourceRecords()
+
+ if init:
+ self.mark_essential()
+ # 1(required), 2(important), 3(standard), 4(optional), 5(extra)
+ self.mark_by_prio(1)
+
+ if srcmode:
+ self.handle_src_list(pkgs)
+ else:
+ self.mark_list(pkgs, crossbuild)
+
+ if controlfile:
+ fobj = open(controlfile, "r")
+
+ try:
+ tagfile = apt_pkg.TagFile(fobj)
+ while tagfile.step() == 1:
+ deps = tagfile.section.get("Build-Depends", "")
+ # Remove extra commas and spaces - apt_pkg.parse_depends doesnt like
+ # lines like ", device-tree-compiler"
+ deps = ', '.join([s.strip() for s in deps.split(',') if s.strip()])
+ print("parsed deps: " + str(deps))
+ for item in apt_pkg.parse_depends(deps, False):
+ pkgname = item[0][0]
+ self.mark_pkg(pkgname, crossbuild)
+
+ finally:
+ fobj.close()
+
+ if init or not srcmode:
+ fetcher = apt_pkg.Acquire(progress)
+ pm = apt_pkg.PackageManager(self.depcache)
+
+ recs = apt_pkg.PackageRecords(self.cache)
+ pm.get_archives(fetcher, sources, recs)
+
+ self.handle_repo(fetcher)
+
+ def load_opts(self):
+ params = {}
+ if os.path.isfile(self.optsfile):
+ with open(self.optsfile, 'rb') as file:
+ data = file.read()
+ if data:
+ params = pickle.loads(data)
+
+ return params
+
+ def save_opts(self, opts):
+ file = open(self.optsfile, 'wb')
+ pickle.dump(opts, file)
+ file.close()
+
+
+class DebRepoArgs(object):
+ def __init__(self):
+ self.workdir = ""
+ self.init = False
+ self.srcmode = False
+ self.controlfile = ""
+ self.aptsrcsfile = ""
+ self.crossbuild = False
+
+ self.opts = {}
+ self.pkgs = []
+
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "", [
+ "init",
+ "srcmode",
+ "workdir=",
+ "repodir=",
+ "repodbdir=",
+ "mirror=",
+ "arch=",
+ "crossarch=",
+ "compatarch=",
+ "distro=",
+ "codename=",
+ "keydir=",
+ "isaraptdir=",
+ "no-check-gpg",
+ "controlfile=",
+ "aptsrcsfile=",
+ "crossbuild"
+ ])
+ except getopt.GetoptError as msg:
+ print("Error: " + str(msg))
+ sys.exit(1)
+
+ for opt, arg in opts:
+ if opt in ("--workdir"):
+ self.workdir = arg
+ if opt in ("--init"):
+ self.init = True
+ if opt in ("--srcmode"):
+ self.srcmode = True
+ if opt in ("--controlfile"):
+ self.controlfile = arg
+ if opt in ("--aptsrcsfile"):
+ self.aptsrcsfile = arg
+ if opt in ("--crossbuild"):
+ self.crossbuild = True
+
+ if opt in ("--repodir",
+ "--repodbdir",
+ "--mirror",
+ "--arch",
+ "--crossarch",
+ "--compatarch",
+ "--distro",
+ "--codename",
+ "--keydir",
+ "--isaraptdir",
+ "--controlfile",
+ ):
+ self.opts[opt[2:]] = arg
+ if opt in ("--no-check-gpg"):
+ self.opts['check_gpg'] = False
+
+ if not self.workdir:
+ print("Error: workdir is not specified")
+ sys.exit(1)
+
+ self.pkgs = args
+
+
+def main():
+ args = DebRepoArgs()
+
+ if not (args.init or args.pkgs or args.controlfile):
+ print("Nothing to do")
+ sys.exit(0)
+
+ if not os.path.exists(args.workdir):
+ os.makedirs(args.workdir)
+
+ debrepo = DebRepo(args.workdir, args.opts)
+
+ if (args.init):
+ debrepo.create_rootfs(args.aptsrcsfile)
+
+ debrepo.apt_config(args.init, args.crossbuild)
+ debrepo.apt_run(args.init, args.srcmode, args.pkgs, args.controlfile, args.crossbuild)
+
+
+if __name__ == "__main__":
+ main()
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 02/13] meta: Add debrepo bbclass handling base-apt prefetching
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 01/13] scripts: Add debrepo python script handling base-apt Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 03/13] meta-isar: Add local ubuntu-focal public key Uladzimir Bely
` (10 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
This class uses 'scripts/debrepo' python script to prefetch given
packages or sources to local base-apt repository.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/debrepo.bbclass | 74 ++++++++++++++++++++++++++++++++++++
meta/conf/bitbake.conf | 5 +++
2 files changed, 79 insertions(+)
create mode 100644 meta/classes/debrepo.bbclass
diff --git a/meta/classes/debrepo.bbclass b/meta/classes/debrepo.bbclass
new file mode 100644
index 00000000..6376b3de
--- /dev/null
+++ b/meta/classes/debrepo.bbclass
@@ -0,0 +1,74 @@
+DEBREPO_WORKDIR ?= "${DEBREPO_TARGET_DIR}"
+
+update_apt_source_list() {
+ chroot_dir=${1}
+ apt_list=${2}
+
+ flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+ sudo -E chroot ${chroot_dir} /usr/bin/apt-get update \
+ -o Dir::Etc::SourceList=\"sources.list.d/${apt_list}.list\" \
+ -o Dir::Etc::SourceParts=\"-\" \
+ -o APT::Get::List-Cleanup=\"0\"
+ "
+}
+
+debrepo_add_packages() {
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ return
+ fi
+
+ args=""
+ if [ "${1}" = "--srcmode" ]; then
+ args="${args} --srcmode"
+ shift
+ fi
+
+ workdir="${1}"
+ pkgs="${2}"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ flock -x "${workdir}/repo.lock" -c "
+ ${SCRIPTSDIR}/debrepo \
+ ${args} \
+ --workdir=\"${workdir}\" \
+ ${pkgs}
+ "
+}
+
+debrepo_handle_controlfile() {
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ return
+ fi
+
+ control_file="${1}"
+
+ args=""
+ pkgs=""
+
+ build_arch=${DISTRO_ARCH}
+ if [ "${ISAR_CROSS_COMPILE}" = "1" ]; then
+ build_arch=${HOST_ARCH}
+ fi
+ if [ "${PACKAGE_ARCH}" != "${build_arch}" ]; then
+ args="--crossbuild"
+ pkgs="${pkgs} crossbuild-essential-${PACKAGE_ARCH}:${build_arch}"
+ pkgs="${pkgs} dose-distcheck:${build_arch}"
+ pkgs="${pkgs} libc-dev:${PACKAGE_ARCH}"
+ pkgs="${pkgs} libstdc++-dev:${PACKAGE_ARCH}"
+ fi
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ flock -x "${DEBREPO_WORKDIR}/repo.lock" -c "
+ ${SCRIPTSDIR}/debrepo \
+ --workdir=\"${DEBREPO_WORKDIR}\" \
+ --controlfile=\"${control_file}\" \
+ ${args} \
+ ${pkgs}
+ "
+}
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index c9f52a86..bea62bb2 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -67,6 +67,11 @@ KERNEL_FILE:mipsel ?= "vmlinux"
KERNEL_FILE:riscv64 ?= "vmlinux"
KERNEL_FILE:arm64 ?= "vmlinux"
+# debrepo config
+DEBREPO_DIR = "${TOPDIR}/debrepo"
+DEBREPO_HOST_DIR = "${DEBREPO_DIR}/${HOST_DISTRO}-${HOST_ARCH}_${DISTRO}-${DISTRO_ARCH}"
+DEBREPO_TARGET_DIR = "${DEBREPO_DIR}/${DISTRO}-${DISTRO_ARCH}"
+
OVERRIDES = "${DISTRO_ARCH}:${COMPAT_OVERRIDE}:${MACHINE}:${DISTRO}:${BASE_DISTRO_CODENAME}:forcevariable"
FILESOVERRIDES = "${DISTRO_ARCH}:${MACHINE}"
COMPAT_OVERRIDE = "${@'compat-arch' if d.getVar('ISAR_ENABLE_COMPAT_ARCH') == '1' else ''}"
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 03/13] meta-isar: Add local ubuntu-focal public key
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 01/13] scripts: Add debrepo python script handling base-apt Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 02/13] meta: Add debrepo bbclass handling base-apt prefetching Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 04/13] meta: Always use base-apt repo in local mode Uladzimir Bely
` (9 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
When debootstrapping Ubuntu from a local repo in signed mode we need
to have a local signing key taken from official Ubuntu repository.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta-isar/conf/distro/ubuntu-focal.conf | 5 +++
meta-isar/conf/distro/ubuntu.public.key | 53 +++++++++++++++++++++++++
2 files changed, 58 insertions(+)
create mode 100644 meta-isar/conf/distro/ubuntu.public.key
diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
index 6292501a..0cb6958d 100644
--- a/meta-isar/conf/distro/ubuntu-focal.conf
+++ b/meta-isar/conf/distro/ubuntu-focal.conf
@@ -13,6 +13,11 @@ HOST_BASE_DISTRO = "${BASE_DISTRO}"
DISTRO_APT_SOURCES:arm64 ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}-ports.list"
HOST_DISTRO_APT_SOURCES:arm64 ?= "conf/distro/${HOST_DISTRO}.list conf/distro/${HOST_DISTRO}-ports.list"
+BOOTSTRAP_KEY = "file://${LAYERDIR_isar}/conf/distro/ubuntu.public.key;sha256sum=36a38199a4bf4eae1e7f574891f7dfcb79b91b87a33a499383265e1224b5e989"
+DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
+HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
+
+
# that is what debootstrap_1.0.118ubuntu1 does anyways
DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
diff --git a/meta-isar/conf/distro/ubuntu.public.key b/meta-isar/conf/distro/ubuntu.public.key
new file mode 100644
index 00000000..994f9f19
--- /dev/null
+++ b/meta-isar/conf/distro/ubuntu.public.key
@@ -0,0 +1,53 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq
+dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ
+HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq
+X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k
+KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns
+AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta
+LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq
+VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd
+yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv
+Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow
+DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB
+tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm
+dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI
+AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH
+beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG
+g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa
+Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy
+q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF
+kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0
+LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr
+BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5
+KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA
+QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp
+SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+
+uOgcXny1UlwtCUzlrSaPmQINBE+tgXgBEADfiL1KNFHT4H4Dw0OR9LemR8ebsFl+
+b9E44IpGhgWYDufj0gaM/UJ1Ti3bHfRT39VVZ6cv1P4mQy0bnAKFbYz/wo+GhzjB
+Wtn6dThYv7n+KL8bptSCXgg1a6en8dCCIA/pwtS2Ut/g4Eu6Z467dvYNlMgCqvg+
+prKIrXf5ibio48j3AFvd1dDJl2cHfyuON35/83vXKXz0FPohQ7N7kPfI+qrlGBYG
+WFzC/QEGje360Q2Yo+rfMoyDEXmPsoZVqf7EE8gjfnXiRqmz/Bg5YQb5bgnGbLGi
+HWtjS+ACIdLUq/h+jlSp57jw8oQktMh2xVMX4utDM0UENeZnPllVJSlR0b+ZmZz7
+paeSar8Yxn4wsNlL7GZbpW5A/WmcmWfuMYoPhBo5Fq1V2/siKNU3UKuf1KH+X0p1
+oZ4oOcZ2bS0Zh3YEG8IQce9Bferq4QMKsekcG9IKS6WBIU7BwaElI2ILD0gSwu8K
+zvNSEeIJhYSsBIEzrWxIBXoN2AC9PCqqXkWlI5Xr/86RWllB3CsoPwEfO8CLJW2L
+lXTen/Fkq4wT+apdhHeiWiSsq/J5OEff0rKHBQ3fK7fyVuVNrJFb2CopaBLyCxTu
+pvxs162jjUNopt0c7OqNBoPoUoVFAxUSpeEwAw6xrM5vROyLMSeh/YnTuRy8WviR
+apZCYo6naTCY5wARAQABtEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmlu
+ZyBLZXkgKDIwMTIpIDxmdHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwECACIFAk+t
+gXgCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEDtP5qzAsh8yXX4QAJHU
+dK6eYMyJcrFP3yKXtUYQMpaHRM/floqZtOFhlmcLVMgBNOr0eLvBU0JcZyZpHMvZ
+ciTDBMWX8ItCYVjRejf0K0lPvHHRGaE7t6JHVUCeznNbDMnOPYVwlVJdZLOa6PmE
+5WXVXpk8uTA8vm6RO2rS23vE7U0pQlV+1GVXMWH4ZLjaQs/Tm7wdvRxeqTbtfOEe
+HGLjmsoh0erHfzMV4wA/9Zq86WzuJS1HxXR6OYDC3/aQX7CxYT1MQxEw/PObnHtk
+l3PRMWdTW7fSQtulEXzpr2/JCev6Mfc8Uy0aD3jng9byVk9GpdNFEjGgaUqjqyZo
+svwAZ4/dmRjmMEibXeNUGC8HeWC3WOVV8L/DiA+miJlwPvwPiA1ZuKBI5A8VF0rN
+HW7QVsG8kQ+PDHgRdsmhpzSRgykN1PgK6UxScKX8LqNKCtKpuEPApka7FQ1u4BoZ
+KjjpBhY1R4TpfFkMIe7qW8XfqoaP99pED3xXch2zFRNHitNJr+yQJH4z/o+2UvnT
+A2niUTHlFSCBoU1MvSq1N2J3qU6oR2cOYJ4ZxqWyCoeQR1x8aPnLlcn4le6HU7To
+cYbHaImcIt7qnG4Ni0OWP4giEhjOpgxtrWgl36mdufvriwya+EHXzn36EvQ9O+bm
+3fyarsnhPe01rlsRxqBiK1JOw/g4GnpX8iLGEX1V
+=kRV1
+-----END PGP PUBLIC KEY BLOCK-----
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 04/13] meta: Always use base-apt repo in local mode
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (2 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 03/13] meta-isar: Add local ubuntu-focal public key Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 05/13] meta: move base-apt from deploy directory to the top Uladzimir Bely
` (8 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
This means only local URLs in apt sources.list* are present during
the build. Any installation of packages is done from local base-apt.
So, base-apt should be always mounted in *_do_mounts since now.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/buildchroot.bbclass | 10 ++--
meta/classes/rootfs.bbclass | 10 ++--
.../isar-bootstrap/isar-bootstrap.inc | 32 ++++++-------
meta/recipes-devtools/base-apt/base-apt.bb | 46 -------------------
4 files changed, 20 insertions(+), 78 deletions(-)
diff --git a/meta/classes/buildchroot.bbclass b/meta/classes/buildchroot.bbclass
index 5abd533b..22a03d45 100644
--- a/meta/classes/buildchroot.bbclass
+++ b/meta/classes/buildchroot.bbclass
@@ -50,13 +50,9 @@ buildchroot_do_mounts() {
mount --rbind /sys '${BUILDCHROOT_DIR}/sys'
mount --make-rslave '${BUILDCHROOT_DIR}/sys'
- # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
- if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}" = 'True' ]
- then
- mkdir -p '${BUILDCHROOT_DIR}/base-apt'
- mountpoint -q '${BUILDCHROOT_DIR}/base-apt' || \
- mount --bind '${REPO_BASE_DIR}' '${BUILDCHROOT_DIR}/base-apt'
- fi
+ mkdir -p '${BUILDCHROOT_DIR}/base-apt'
+ mountpoint -q '${BUILDCHROOT_DIR}/base-apt' || \
+ mount --bind '${REPO_BASE_DIR}' '${BUILDCHROOT_DIR}/base-apt'
# Refresh or remove /etc/resolv.conf at this chance
if [ "${@repr(bb.utils.to_boolean(d.getVar('BB_NO_NETWORK')))}" = 'True' ]
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 0eed3d02..92d665db 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -54,13 +54,9 @@ rootfs_do_mounts() {
mount --bind '${REPO_ISAR_DIR}/${DISTRO}' '${ROOTFSDIR}/isar-apt'
fi
- # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
- if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}" = 'True' ]
- then
- mkdir -p '${ROOTFSDIR}/base-apt'
- mountpoint -q '${ROOTFSDIR}/base-apt' || \
- mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
- fi
+ mkdir -p '${ROOTFSDIR}/base-apt'
+ mountpoint -q '${ROOTFSDIR}/base-apt' || \
+ mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
EOSUDO
}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 99d75e21..01668b93 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -42,10 +42,9 @@ python () {
# installation afterwards. However, debootstrap will include the key into
# the rootfs automatically thus the right place is distro_bootstrap_keys.
- if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- own_pub_key = d.getVar("BASE_REPO_KEY")
- if own_pub_key:
- distro_bootstrap_keys += own_pub_key.split()
+ own_pub_key = d.getVar("BASE_REPO_KEY")
+ if own_pub_key:
+ distro_bootstrap_keys += own_pub_key.split()
for key in distro_bootstrap_keys:
d.appendVar("SRC_URI", " %s" % key)
@@ -284,7 +283,7 @@ do_bootstrap() {
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
fi
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z "${BASE_REPO_KEY}" ]; then
+ if [ -z "${BASE_REPO_KEY}" ]; then
debootstrap_args="$debootstrap_args --no-check-gpg"
fi
E="${@ isar_export_proxies(d)}"
@@ -311,24 +310,21 @@ do_bootstrap() {
install -v -m644 "${APTPREFS}" \
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
+ line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
if [ -z "${BASE_REPO_KEY}" ]; then
line="[trusted=yes] ${line}"
fi
- echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
+ mkdir -p ${ROOTFSDIR}/base-apt
+ mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- mkdir -p ${ROOTFSDIR}/base-apt
- mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- else
- install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- fi
install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
rm -f "${ROOTFSDIR}/etc/apt/sources.list"
rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index ea885fe6..d5b3cf4e 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -10,43 +10,6 @@ SRC_URI = "file://distributions.in"
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-BASE_REPO_FEATURES ?= ""
-
-populate_base_apt() {
- base_distro="${1}"
-
- find "${DEBDIR}"/"${base_distro}-${BASE_DISTRO_CODENAME}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is already in base-apt
- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${base_distro}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${base_distro}" \
- "${REPO_BASE_DB_DIR}"/"${base_distro}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${base_distro}" \
- "${REPO_BASE_DB_DIR}"/"${base_distro}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${base_distro}-${BASE_DISTRO_CODENAME}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${base_distro}" \
- "${REPO_BASE_DB_DIR}"/"${base_distro}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
do_cache[stamp-extra-info] = "${DISTRO}"
do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
@@ -57,9 +20,6 @@ repo() {
"${BASE_DISTRO_CODENAME}" \
"${WORKDIR}/distributions.in" \
"${KEYFILES}"
- populate_base_apt "${BASE_DISTRO}"
- repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}"
if [ '${BASE_DISTRO}' != '${HOST_BASE_DISTRO}' ]; then
repo_create "${REPO_BASE_DIR}"/"${HOST_BASE_DISTRO}" \
@@ -67,16 +27,10 @@ repo() {
"${BASE_DISTRO_CODENAME}" \
"${WORKDIR}/distributions.in" \
"${KEYFILES}"
- populate_base_apt "${HOST_BASE_DISTRO}"
- repo_sanity_test "${REPO_BASE_DIR}"/"${HOST_BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${HOST_BASE_DISTRO}"
fi
}
python do_cache() {
- if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- return 0
-
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 05/13] meta: move base-apt from deploy directory to the top
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (3 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 04/13] meta: Always use base-apt repo in local mode Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 06/13] meta: Use cached base-apt repo to debootstrap Uladzimir Bely
` (7 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
While base-apt is supposed to be always used for deboostrapping and
installing build dependencies, move it out of temporary directory.
This also allows not to change some of CI tests that completely
remove TMPDIR and underlaying DEPLOY_DIR.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/conf/bitbake.conf | 4 ++--
testsuite/citest.py | 3 +++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index bea62bb2..361bf1f0 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -98,8 +98,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
# Base apt repository paths
-REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${TOPDIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${TOPDIR}/base-apt/${DISTRO}/db"
# Setup our default hash policy
BB_SIGNATURE_HANDLER ?= "OEBasicHash"
diff --git a/testsuite/citest.py b/testsuite/citest.py
index 17a90244..2393e275 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -86,9 +86,11 @@ class ReproTest(CIBaseTest):
self.init()
try:
+ self.delete_from_build_dir('base-apt')
self.perform_repro_test(targets, signed=True)
finally:
self.move_in_build_dir('tmp', 'tmp_repro_signed')
+ self.move_in_build_dir('base-apt', 'base-apt_repro_signed')
def test_repro_unsigned(self):
targets = [
@@ -101,6 +103,7 @@ class ReproTest(CIBaseTest):
self.perform_repro_test(targets, cross=False)
finally:
self.move_in_build_dir('tmp', 'tmp_repro_unsigned')
+ self.move_in_build_dir('base-apt', 'base-apt_repro_unsigned')
class CcacheTest(CIBaseTest):
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 06/13] meta: Use cached base-apt repo to debootstrap
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (4 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 05/13] meta: move base-apt from deploy directory to the top Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 07/13] meta: Setup debrepo context for dpkg-base Uladzimir Bely
` (6 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
This patch makes local base-apt repo to be created before
debootstrap task. So, debootstrap is then done from it.
The required packages are downloaded via python-apt and
reprepro creates debian-like repository from .deb files.
For debian targets host keyring is used while ubuntu/raspbian
targets use keys specified by DISTRO_BOOTSTRAP_KEYS variable.
The goal is have workable base-apt repo before first build completed.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
.../isar-bootstrap/isar-bootstrap.inc | 92 ++++++++++++++++---
1 file changed, 78 insertions(+), 14 deletions(-)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 01668b93..4a3ef919 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -29,6 +29,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
+BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
+
FILESEXTRAPATHS:append = ":${BBPATH}"
inherit deb-dl-dir
@@ -273,12 +275,69 @@ do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+inherit debrepo
+
+debrepo_bootstrap_prepare() {
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ return
+ fi
+
+ debrepo_args=""
+ if [ "${BASE_DISTRO}" != "debian" ]; then
+ if [ "${BASE_DISTRO}" != "raspbian" ] && [ "${BASE_DISTRO}" != "raspios" ] || [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
+ debrepo_args="$debrepo_args --keydir=${WORKDIR}"
+ fi
+ else
+ if [ "${BASE_DISTRO_CODENAME}" = "sid" ]; then
+ debrepo_args="$debrepo_args --keydir=${WORKDIR}"
+ fi
+ fi
+ #if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ debrepo_args="$debrepo_args --no-check-gpg"
+ #fi
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ debrepo_args="$debrepo_args --compatarch=${COMPAT_DISTRO_ARCH}"
+ fi
+
+ debrepo_workdir=${DEBREPO_TARGET_DIR}
+ if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
+ debrepo_args="$debrepo_args --crossarch=${DISTRO_ARCH}"
+ if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+ debrepo_workdir=${DEBREPO_HOST_DIR}
+ fi
+ else
+ debrepo_args="$debrepo_args --isaraptdir=${REPO_ISAR_DIR}/${DISTRO}"
+ fi
+ mkdir -p "${DEBREPO_WORKDIR}"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ flock -x "${DEBREPO_WORKDIR}/repo.lock" -c "
+ ${SCRIPTSDIR}/debrepo --init \
+ --workdir=\"${debrepo_workdir}\" \
+ --aptsrcsfile=\"${APTSRCS_INIT}\" \
+ --repodir=\"${REPO_BASE_DIR}\" \
+ --repodbdir=\"${REPO_BASE_DB_DIR}\" \
+ --mirror=\"${@get_distro_source(d)}\" \
+ --arch=\"${BOOTSTRAP_DISTRO_ARCH}\" \
+ --distro=\"${BOOTSTRAP_BASE_DISTRO}\" \
+ --codename=\"${BASE_DISTRO_CODENAME}\" \
+ ${debrepo_args} \
+ gnupg locales usr-is-merged
+ "
+}
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
bbfatal "${DISTRO_ARCH} does not have a compat arch"
fi
fi
+
+ debrepo_bootstrap_prepare
+
debootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
@@ -292,19 +351,21 @@ do_bootstrap() {
sudo rm -rf --one-file-system "${ROOTFSDIR}"
deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ (flock 9
sudo -E -s <<'EOSUDO'
set -e
if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
- arch_param="--arch=${DISTRO_ARCH}"
+ arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH}"
fi
- ${DEBOOTSTRAP} $debootstrap_args \
- $arch_param \
- ${@get_distro_components_argument(d)} \
- "${@get_distro_suite(d)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d)}" \
- ${DISTRO_DEBOOTSTRAP_SCRIPT}
-
+ flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+ ${DEBOOTSTRAP} $debootstrap_args \
+ $arch_param \
+ ${@get_distro_components_argument(d)} \
+ ${@get_distro_suite(d)} \
+ ${ROOTFSDIR} \
+ file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO} \
+ ${DISTRO_DEBOOTSTRAP_SCRIPT}
+ "
# Install apt config
mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
install -v -m644 "${APTPREFS}" \
@@ -373,11 +434,13 @@ do_bootstrap() {
chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
fi
- chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
- -o APT::Update::Error-Mode=any
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
- chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
- -o Debug::pkgProblemResolver=yes
+ flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+ chroot ${ROOTFSDIR} /usr/bin/apt-get update -y \
+ -o APT::Update::Error-Mode=any
+ chroot ${ROOTFSDIR} /usr/bin/apt-get install -y -f
+ chroot ${ROOTFSDIR} /usr/bin/apt-get dist-upgrade -y \
+ -o Debug::pkgProblemResolver=yes
+ "
umount -l "${ROOTFSDIR}/dev/shm"
umount -l "${ROOTFSDIR}/dev/pts"
@@ -389,6 +452,7 @@ do_bootstrap() {
# Finalize debootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
+ ) 9>"${DEBREPO_WORKDIR}/repo.lock"
deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
# Cleanup apt cache
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 07/13] meta: Setup debrepo context for dpkg-base
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (5 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 06/13] meta: Use cached base-apt repo to debootstrap Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 08/13] meta: Setup debrepo context for buildchroot Uladzimir Bely
` (5 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
When working with dpkg-base context, we need to consider host/target
and cross/native modes. So, debrepo defaults should be overriden in
these cases.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/dpkg-base.bbclass | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index ad28f7b3..eb4ac34a 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -10,6 +10,17 @@ inherit terminal
inherit repository
inherit deb-dl-dir
+python __anonymous() {
+ distro_arch = d.getVar('DISTRO_ARCH')
+ host_arch = d.getVar('HOST_ARCH')
+ cross = d.getVar('ISAR_CROSS_COMPILE', True)
+
+ if cross == "0" or host_arch == distro_arch:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR', True))
+ else:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True))
+}
+
DEPENDS ?= ""
RPROVIDES ?= "${PROVIDES}"
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 08/13] meta: Setup debrepo context for buildchroot
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (6 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 07/13] meta: Setup debrepo context for dpkg-base Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 09/13] meta: Setup debrepo context for sbuild-chroot Uladzimir Bely
` (4 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
When working with buildchroot context, we need to consider host/target
and cross/native modes. So, debrepo defaults should be overriden
in these cases.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/recipes-devtools/buildchroot/buildchroot.inc | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
index f74896fb..320df55a 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.inc
+++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
@@ -13,6 +13,17 @@ SRC_URI = "file://configscript.sh \
file://deps.sh"
PV = "1.0"
+python __anonymous() {
+ distro_arch = d.getVar('DISTRO_ARCH')
+ host_arch = d.getVar('HOST_ARCH')
+ variant = d.getVar('BUILDCHROOT_VARIANT', True)
+
+ if variant == "target" or host_arch == distro_arch:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR', True))
+ else:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True))
+}
+
inherit rootfs
BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 09/13] meta: Setup debrepo context for sbuild-chroot
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (7 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 08/13] meta: Setup debrepo context for buildchroot Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 10/13] base-apt: Predownload packages to base-apt before install Uladzimir Bely
` (3 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
When working with sbuild-chroot debrepo context, we need to consider
host/target and cross/native modes. So, debrepo defaults should be
overriden in these cases.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
index fb061dac..1450b764 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
@@ -8,6 +8,17 @@ LIC_FILES_CHKSUM = "file://${LAYERDIR_core}/licenses/COPYING.GPLv2;md5=751419260
PV = "1.0"
+python __anonymous() {
+ distro_arch = d.getVar('DISTRO_ARCH')
+ host_arch = d.getVar('HOST_ARCH')
+ variant = d.getVar('SBUILD_VARIANT', True)
+
+ if variant == "target" or host_arch == distro_arch:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR', True))
+ else:
+ d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True))
+}
+
inherit rootfs
python() {
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 10/13] base-apt: Predownload packages to base-apt before install
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (8 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 09/13] meta: Setup debrepo context for sbuild-chroot Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 11/13] meta: Add cache-deb-src functionality to base-apt Uladzimir Bely
` (2 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
This patch uses debrepo script to predownload packages to base-apt
repository before they are installed in buildchroot/rootfs.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/dpkg-base.bbclass | 31 ++++++++++++++++++--
meta/classes/dpkg.bbclass | 4 ++-
meta/classes/image-locales-extension.bbclass | 5 ++++
meta/classes/image-tools-extension.bbclass | 9 ++++++
meta/classes/rootfs.bbclass | 4 +++
5 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index eb4ac34a..ee745ee1 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -21,6 +21,8 @@ python __anonymous() {
d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True))
}
+inherit debrepo
+
DEPENDS ?= ""
RPROVIDES ?= "${PROVIDES}"
@@ -118,16 +120,30 @@ do_apt_fetch() {
E="${@ isar_export_proxies(d)}"
schroot_create_configs
+ debrepo_add_packages --srcmode "${DEBREPO_TARGET_DIR}" "${SRC_APT}"
+
schroot_cleanup() {
schroot_delete_configs
}
trap 'exit 1' INT HUP QUIT TERM ALRM USR1
trap 'schroot_cleanup' EXIT
+ # Begin chroot session
+ session_id=$(schroot -b -c ${SBUILD_CHROOT})
+
+ flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+ schroot -d / -r -c ${session_id} -u root -- \
+ sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+ "
+
for uri in "${SRC_APT}"; do
- schroot -d / -c ${SBUILD_CHROOT} -- \
+ schroot -d / -r -c ${session_id} -- \
sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}"
done
+
+ # End chroot session
+ schroot -e -c ${session_id}
+
schroot_delete_configs
}
@@ -148,8 +164,16 @@ do_apt_unpack() {
trap 'exit 1' INT HUP QUIT TERM ALRM USR1
trap 'schroot_cleanup' EXIT
+ # Begin chroot session
+ session_id=$(schroot -b -c ${SBUILD_CHROOT})
+
+ flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+ schroot -d / -r -c ${session_id} -u root -- \
+ sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+ "
+
for uri in "${SRC_APT}"; do
- schroot -d / -c ${SBUILD_CHROOT} -- \
+ schroot -d / -r -c ${session_id} -- \
sh -c ' \
set -e
dscfile="$(apt-get -y -qq --print-uris --only-source source "${2}" | cut -d " " -f2 | grep -E "*.dsc")"
@@ -158,6 +182,9 @@ do_apt_unpack() {
dpkg-source -x "${dscfile}" "${PPS}"' \
my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}"
done
+
+ # End chroot session
+ schroot -e -c ${session_id}
schroot_delete_configs
}
do_apt_unpack[network] = "${TASK_USE_SUDO}"
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index d5285032..b3d03516 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -32,6 +32,7 @@ dpkg_runbuild() {
export PARALLEL_MAKE="${PARALLEL_MAKE}"
rm -f ${SBUILD_CONFIG}
+ debrepo_handle_controlfile "${WORKDIR}/${PPS}/debian/control"
env | while read -r line; do
# Filter the same lines
@@ -104,11 +105,12 @@ dpkg_runbuild() {
sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \
--host=${PACKAGE_ARCH} --build=${SBUILD_HOST_ARCH} ${profiles} \
--no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \
- --no-apt-update \
+ --no-apt-update --no-apt-upgrade --no-apt-distupgrade \
--chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \
--chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
--chroot-setup-commands="rm -f /var/log/dpkg.log" \
--chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \
+ --chroot-setup-commands="flock -x /base-apt/repo.lock -c 'apt-get -y update'" \
--finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
--finished-build-commands="cp -Ln --no-preserve=owner ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \
--finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 65b9ac80..4bdcb39d 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -6,6 +6,8 @@
# This class extends the image.bbclass for setting locales and purging unneeded
# ones.
+inherit debrepo
+
LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
en_US ISO-8859-1\n"
LOCALE_DEFAULT ?= "en_US.UTF-8"
@@ -29,6 +31,9 @@ ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
image_install_localepurge_download() {
+ debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge"
+ update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
sudo -E chroot '${ROOTFSDIR}' \
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only localepurge
}
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 2d3dda4f..a74820b8 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -6,6 +6,7 @@
# This file extends the image.bbclass to supply tools for futher imager functions
inherit buildchroot
+inherit debrepo
IMAGER_INSTALL ??= ""
IMAGER_BUILD_DEPS ??= ""
@@ -21,14 +22,22 @@ do_install_imager_deps() {
fi
distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ debrepo_workdir=${DEBREPO_TARGET_DIR}
if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+ debrepo_workdir=${DEBREPO_HOST_DIR}
+ fi
fi
buildchroot_do_mounts
E="${@ isar_export_proxies(d)}"
deb_dl_dir_import ${BUILDCHROOT_DIR} ${distro}
+
+ debrepo_add_packages "${debrepo_workdir}" "${IMAGER_INSTALL}"
+ update_apt_source_list "${BUILDCHROOT_DIR}" "base-apt"
+
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get update \
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 92d665db..a3aec04c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -2,6 +2,7 @@
# Copyright (c) Siemens AG, 2020
inherit deb-dl-dir
+inherit debrepo
ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
@@ -143,6 +144,9 @@ rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
rootfs_install_pkgs_download() {
+ debrepo_add_packages "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}"
+ update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
sudo -E chroot '${ROOTFSDIR}' \
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 11/13] meta: Add cache-deb-src functionality to base-apt
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (9 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 10/13] base-apt: Predownload packages to base-apt before install Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 12/13] meta: Specify grub-efi packages arch Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 13/13] isar-apt: Fix copying isar-apt to workdir Uladzimir Bely
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/deb-dl-dir.bbclass | 22 ++++++++++++++++++++++
meta/classes/rootfs.bbclass | 4 ++++
2 files changed, 26 insertions(+)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index ca2a1ee2..90ae1dec 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -43,6 +43,28 @@ debsrc_undo_mounts() {
EOSUDO
}
+debsrc_fill_base_apt() {
+ export rootfs="$1"
+
+ find "${REPO_BASE_DIR}" -maxdepth 6 -type f -iname '*\.deb' | while read package; do
+ is_not_part_of_current_build "${package}" && continue
+ # Get source package name if available, fallback to package name
+ local src="$( dpkg-deb --field "${package}" Source | awk '{printf $1}' )"
+ [ -z "$src" ] && src="$( dpkg-deb --field "${package}" Package )"
+ # Get source package version if available, fallback to package version
+ local version="$( dpkg-deb --field "${package}" Source | awk '{gsub(/[()]/,""); printf $2}')"
+ [ -z "$version" ] && version="$( dpkg-deb --field "${package}" Version )"
+ # TODO: get back to the code below when debian bug #1004372 is fixed
+ # local src="$( dpkg-deb --show --showformat '${source:Package}' "${package}" )"
+ # local version="$( dpkg-deb --show --showformat '${source:Version}' "${package}" )"
+ local dscname="$(echo ${src}_${version} | sed -e 's/_[0-9]\+:/_/')"
+ local dscfile=$(find "${DEBSRCDIR}"/"${rootfs_distro}" -name "${dscname}.dsc")
+ [ -n "$dscfile" ] && continue
+
+ debrepo_add_packages --srcmode "${DEBREPO_TARGET_DIR}" "${src}=${version}"
+ done
+}
+
debsrc_download() {
export rootfs="$1"
export rootfs_distro="$2"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index a3aec04c..ba41006d 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -225,8 +225,12 @@ cache_deb_src() {
sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
+
+ debsrc_fill_base_apt ${ROOTFSDIR}
+ update_apt_source_list "${ROOTFSDIR}" "base-apt"
debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
+
sudo rm -f "${ROOTFSDIR}"/etc/resolv.conf
if [ -e "${ROOTFSDIR}"/etc/resolv.conf.isar ] ||
[ -h "${ROOTFSDIR}"/etc/resolv.conf.isar ]; then
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 12/13] meta: Specify grub-efi packages arch
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (10 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 11/13] meta: Add cache-deb-src functionality to base-apt Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
2023-02-22 6:41 ` [PATCH v4 13/13] isar-apt: Fix copying isar-apt to workdir Uladzimir Bely
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
When using python-apt, it can't mark it for downloading when used
without ":<arch>" field, so we miss the package in base-apt repo.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/conf/distro/debian-common.conf | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/meta/conf/distro/debian-common.conf b/meta/conf/distro/debian-common.conf
index 5610dcd4..ee84b7dd 100644
--- a/meta/conf/distro/debian-common.conf
+++ b/meta/conf/distro/debian-common.conf
@@ -23,10 +23,10 @@ WIC_IMAGER_INSTALL = "parted \
tar \
fdisk"
-GRUB_BOOTLOADER_INSTALL:amd64 = "grub-efi-amd64-bin"
-GRUB_BOOTLOADER_INSTALL:i386 = "grub-efi-ia32-bin"
-GRUB_BOOTLOADER_INSTALL:armhf = "grub-efi-arm-bin"
-GRUB_BOOTLOADER_INSTALL:arm64 = "grub-efi-arm64-bin"
+GRUB_BOOTLOADER_INSTALL:amd64 = "grub-efi-amd64-bin:amd64"
+GRUB_BOOTLOADER_INSTALL:i386 = "grub-efi-ia32-bin:i386"
+GRUB_BOOTLOADER_INSTALL:armhf = "grub-efi-arm-bin:armhf"
+GRUB_BOOTLOADER_INSTALL:arm64 = "grub-efi-arm64-bin:arm64"
GRUB_DEBIAN_SB_CHAIN:amd64 = "grub-efi-amd64-signed shim-signed"
GRUB_DEBIAN_SB_MOK:amd64 = "shim-helpers-amd64-signed"
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH v4 13/13] isar-apt: Fix copying isar-apt to workdir
2023-02-22 6:41 [PATCH v4 00/13] Improving base-apt usage PoC Uladzimir Bely
` (11 preceding siblings ...)
2023-02-22 6:41 ` [PATCH v4 12/13] meta: Specify grub-efi packages arch Uladzimir Bely
@ 2023-02-22 6:41 ` Uladzimir Bely
12 siblings, 0 replies; 14+ messages in thread
From: Uladzimir Bely @ 2023-02-22 6:41 UTC (permalink / raw)
To: isar-users
Using hardlinks (cp -l) for "dists" may lead to a broken isar-apt
state when some parallel process changes original files in deploy
directory and this is reflected in isar-apt in workdir.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta/classes/dpkg-base.bbclass | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index ee745ee1..e50e55de 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -227,9 +227,12 @@ do_prepare_build[depends] = "${SCHROOT_DEP}"
do_prepare_build:append() {
# Make a local copy of isar-apt repo that is not affected by other parallel builds
- mkdir -p ${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}
- rm -rf ${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/*
- cp -Rl ${REPO_ISAR_DIR} ${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}
+ rm -rf "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/*"
+ mkdir -p "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}"
+ cp -Rf "${REPO_ISAR_DIR}/${DISTRO}/dists" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
+ if [ -d "${REPO_ISAR_DIR}/${DISTRO}/pool" ]; then
+ cp -Rlf "${REPO_ISAR_DIR}/${DISTRO}/pool" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
+ fi
}
do_prepare_build[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
--
2.20.1
^ permalink raw reply [flat|nested] 14+ messages in thread