* [PATCH v2 1/1] [meta] Use cached base-apt repo to debootstrap
2021-10-01 9:13 [PATCH v2 0/1] Improving base-apt usage Uladzimir Bely
@ 2021-10-01 9:13 ` Uladzimir Bely
2021-10-05 8:36 ` [PATCH v2 0/1] Improving base-apt usage Henning Schild
1 sibling, 0 replies; 3+ messages in thread
From: Uladzimir Bely @ 2021-10-01 9:13 UTC (permalink / raw)
To: isar-users
This patch makes local base-apt repo to be created before
debootstrap task. So, debootstrap is then done from it.
The required packages are downloaded via python-apt and
reprepro creates debian-like repository from .deb files.
For debian targets host keyring is used while ubuntu/raspbian
targets use keys specified by DISTRO_BOOTSTRAP_KEYS variable.
The goal is have workable base-apt repo before first build completed.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta-isar/conf/distro/ubuntu-focal.conf | 4 +
meta-isar/conf/distro/ubuntu.public.key | 53 +++++
.../isar-bootstrap/isar-bootstrap.inc | 35 ++-
scripts/debrepo.py | 216 ++++++++++++++++++
4 files changed, 307 insertions(+), 1 deletion(-)
create mode 100644 meta-isar/conf/distro/ubuntu.public.key
create mode 100644 scripts/debrepo.py
diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
index 4dfa201..67a59f6 100644
--- a/meta-isar/conf/distro/ubuntu-focal.conf
+++ b/meta-isar/conf/distro/ubuntu-focal.conf
@@ -11,6 +11,10 @@ DISTRO_APT_SOURCES ?= "conf/distro/${DISTRO}.list"
DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${DISTRO}-ports.list"
HOST_DISTRO_APT_SOURCES_append_arm64 = " ${DISTRO_APT_SOURCES}"
+BOOTSTRAP_KEY = "file://${LAYERDIR_isar}/conf/distro/ubuntu.public.key;sha256sum=36a38199a4bf4eae1e7f574891f7dfcb79b91b87a33a499383265e1224b5e989"
+DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
+HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
+
BASE_DISTRO_CODENAME = "focal"
# that is what debootstrap_1.0.118ubuntu1 does anyways
diff --git a/meta-isar/conf/distro/ubuntu.public.key b/meta-isar/conf/distro/ubuntu.public.key
new file mode 100644
index 0000000..994f9f1
--- /dev/null
+++ b/meta-isar/conf/distro/ubuntu.public.key
@@ -0,0 +1,53 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq
+dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ
+HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq
+X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k
+KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns
+AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta
+LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq
+VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd
+yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv
+Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow
+DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB
+tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm
+dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI
+AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH
+beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG
+g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa
+Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy
+q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF
+kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0
+LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr
+BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5
+KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA
+QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp
+SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+
+uOgcXny1UlwtCUzlrSaPmQINBE+tgXgBEADfiL1KNFHT4H4Dw0OR9LemR8ebsFl+
+b9E44IpGhgWYDufj0gaM/UJ1Ti3bHfRT39VVZ6cv1P4mQy0bnAKFbYz/wo+GhzjB
+Wtn6dThYv7n+KL8bptSCXgg1a6en8dCCIA/pwtS2Ut/g4Eu6Z467dvYNlMgCqvg+
+prKIrXf5ibio48j3AFvd1dDJl2cHfyuON35/83vXKXz0FPohQ7N7kPfI+qrlGBYG
+WFzC/QEGje360Q2Yo+rfMoyDEXmPsoZVqf7EE8gjfnXiRqmz/Bg5YQb5bgnGbLGi
+HWtjS+ACIdLUq/h+jlSp57jw8oQktMh2xVMX4utDM0UENeZnPllVJSlR0b+ZmZz7
+paeSar8Yxn4wsNlL7GZbpW5A/WmcmWfuMYoPhBo5Fq1V2/siKNU3UKuf1KH+X0p1
+oZ4oOcZ2bS0Zh3YEG8IQce9Bferq4QMKsekcG9IKS6WBIU7BwaElI2ILD0gSwu8K
+zvNSEeIJhYSsBIEzrWxIBXoN2AC9PCqqXkWlI5Xr/86RWllB3CsoPwEfO8CLJW2L
+lXTen/Fkq4wT+apdhHeiWiSsq/J5OEff0rKHBQ3fK7fyVuVNrJFb2CopaBLyCxTu
+pvxs162jjUNopt0c7OqNBoPoUoVFAxUSpeEwAw6xrM5vROyLMSeh/YnTuRy8WviR
+apZCYo6naTCY5wARAQABtEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmlu
+ZyBLZXkgKDIwMTIpIDxmdHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwECACIFAk+t
+gXgCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEDtP5qzAsh8yXX4QAJHU
+dK6eYMyJcrFP3yKXtUYQMpaHRM/floqZtOFhlmcLVMgBNOr0eLvBU0JcZyZpHMvZ
+ciTDBMWX8ItCYVjRejf0K0lPvHHRGaE7t6JHVUCeznNbDMnOPYVwlVJdZLOa6PmE
+5WXVXpk8uTA8vm6RO2rS23vE7U0pQlV+1GVXMWH4ZLjaQs/Tm7wdvRxeqTbtfOEe
+HGLjmsoh0erHfzMV4wA/9Zq86WzuJS1HxXR6OYDC3/aQX7CxYT1MQxEw/PObnHtk
+l3PRMWdTW7fSQtulEXzpr2/JCev6Mfc8Uy0aD3jng9byVk9GpdNFEjGgaUqjqyZo
+svwAZ4/dmRjmMEibXeNUGC8HeWC3WOVV8L/DiA+miJlwPvwPiA1ZuKBI5A8VF0rN
+HW7QVsG8kQ+PDHgRdsmhpzSRgykN1PgK6UxScKX8LqNKCtKpuEPApka7FQ1u4BoZ
+KjjpBhY1R4TpfFkMIe7qW8XfqoaP99pED3xXch2zFRNHitNJr+yQJH4z/o+2UvnT
+A2niUTHlFSCBoU1MvSq1N2J3qU6oR2cOYJ4ZxqWyCoeQR1x8aPnLlcn4le6HU7To
+cYbHaImcIt7qnG4Ni0OWP4giEhjOpgxtrWgl36mdufvriwya+EHXzn36EvQ9O+bm
+3fyarsnhPe01rlsRxqBiK1JOw/g4GnpX8iLGEX1V
+=kRV1
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index b8af676..6c762bc 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -271,6 +271,37 @@ do_bootstrap[vardeps] += " \
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
+do_localrepo[lockfiles] = "${TMPDIR}/debrepo/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}/isar.lock"
+do_localrepo[depends] = "base-apt:do_cache isar-apt:do_cache_config"
+do_localrepo() {
+ debrepo_args=""
+ if [ "${BASE_DISTRO}" != "debian" ]; then
+ if [ "${BASE_DISTRO}" != "raspbian" ] || [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
+ debrepo_args="$debrepo_args --keydir=${WORKDIR}"
+ fi
+ fi
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ debrepo_args="$debrepo_args --no-check-gpg"
+ fi
+ if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
+ arch_param="--arch=${DISTRO_ARCH}"
+ else
+ arch_param="--arch=${HOST_ARCH}"
+ fi
+ python3 "${SCRIPTSDIR}/debrepo.py" \
+ --workdir="${TMPDIR}/debrepo/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" \
+ --repodir="${REPO_BASE_DIR}" \
+ --repodbdir="${REPO_BASE_DB_DIR}" \
+ --mirror="${@get_distro_source(d)}" \
+ ${arch_param} \
+ --distro="${BASE_DISTRO}" \
+ --codename="${BASE_DISTRO_CODENAME}" \
+ --components="main,contrib,non-free" \
+ ${debrepo_args} \
+ gnupg locales
+ #deb_dl_dir_export "${TMPDIR}/debrepo/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${BOOTSTRAP_DISTRO}"
+}
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -293,6 +324,7 @@ do_bootstrap() {
sudo rm -rf --one-file-system "${ROOTFSDIR}"
deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_DISTRO}"
+ debootstrap_args="$debootstrap_args --no-check-gpg"
sudo -E -s <<'EOSUDO'
set -e
if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
@@ -303,7 +335,7 @@ do_bootstrap() {
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${ROOTFSDIR}" \
- "${@get_distro_source(d)}" \
+ "file://${REPO_BASE_DIR}/${BASE_DISTRO}" \
${DISTRO_DEBOOTSTRAP_SCRIPT}
# Install apt config
@@ -398,6 +430,7 @@ EOSUDO
}
addtask bootstrap before do_build after do_generate_keyrings
+addtask localrepo before do_bootstrap after do_generate_keyrings
CLEANFUNCS = "clean_deploy"
clean_deploy() {
diff --git a/scripts/debrepo.py b/scripts/debrepo.py
new file mode 100644
index 0000000..ae7369a
--- /dev/null
+++ b/scripts/debrepo.py
@@ -0,0 +1,216 @@
+#!/usr/bin/python3
+
+import os
+import sys
+
+import subprocess
+import getopt
+
+import apt_pkg
+import apt.progress.base
+
+class DebRepo(object):
+ def __init__(self, workdir, repodir, repodbdir, mirror, arch, distro, codename, components, keydir, check_gpg):
+ self.mirror = mirror
+ self.arch = arch
+ self.distro = distro
+ self.codename = codename
+ self.components = components.replace(",", " ")
+
+ self.rootfs = workdir
+ self.repo = repodir + "/" + distro
+ self.repodb = repodbdir + "/" + distro
+
+ self.keydir = keydir
+ self.check_gpg = check_gpg
+
+ self.cache = None
+ self.depcache = None
+
+
+ def create_rootfs(self):
+ if not os.path.exists(self.rootfs + "/var/lib/dpkg"):
+ os.makedirs(self.rootfs + "/var/lib/dpkg")
+ f = open(self.rootfs + "/var/lib/dpkg" + "/status", "w")
+ f.flush
+ f.close
+
+ if not os.path.exists(self.rootfs + "/etc/apt/"):
+ os.makedirs(self.rootfs + "/etc/apt/")
+ f = open(self.rootfs + "/etc/apt/sources.list", "w")
+ f.write("deb [arch=" + self.arch + "] " + self.mirror + " " + self.codename + " " + self.components + "\n")
+ f.flush
+ f.close
+
+ if not os.path.exists(self.rootfs + "/var/cache/apt/archives/partial"):
+ os.makedirs(self.rootfs + "/var/cache/apt/archives/partial")
+
+
+ def create_repo_dist(self):
+ if not os.path.exists(self.repo + "/conf"):
+ os.makedirs(self.repo + "/conf")
+ f = open(self.repo + "/conf" + "/distributions", "w")
+ f.write("Codename: " + self.codename + "\n")
+ f.write("Architectures: i386 armhf arm64 amd64 mipsel riscv64 source" + "\n")
+ f.write("Components: " + self.components + "\n")
+ f.flush
+ f.close
+
+
+ def apt_config(self):
+ # Configure apt to work with empty directory
+ apt_pkg.config.set("APT::Architecture", self.arch)
+ apt_pkg.config.set("Dir", self.rootfs)
+ apt_pkg.config.set("Dir::Cache", self.rootfs + "/var/cache/apt")
+ apt_pkg.config.set("Dir::State::status", self.rootfs + "/var/lib/dpkg/status")
+
+ apt_pkg.config.set("APT::Install-Recommends", "0")
+ apt_pkg.config.set("APT::Install-Suggests", "0")
+
+ # Use host keys for authentification
+ #apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
+ #apt_pkg.config.set("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d")
+ apt_pkg.config.set("Dir::Etc::TrustedParts", self.keydir)
+
+ # Allow using repositories without keys
+ if not self.check_gpg:
+ apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
+
+
+ def mark_essential(self):
+ for pkg in self.cache.packages:
+ if pkg.essential:
+ self.depcache.mark_install(pkg)
+
+
+ def mark_by_prio(self, priority):
+ for pkg in self.cache.packages:
+ ver = self.depcache.get_candidate_ver(pkg)
+ if ver and ver.priority <= priority:
+ self.depcache.mark_install(pkg)
+
+
+ def mark_list(self, pkglist):
+ if pkglist:
+ for pkgname in pkglist:
+ pkg = self.cache[pkgname]
+ self.depcache.mark_install(pkg)
+
+
+ def handle_deb(self, item):
+ subprocess.run([
+ "reprepro",
+ "--dbdir", self.repodb,
+ "--outdir", self.repo,
+ "--confdir", self.repo + "/conf",
+ "-C", "main",
+ "includedeb",
+ self.codename,
+ item.destfile
+ ])
+
+ def handle_repo(self, fetcher):
+ fetcher.run()
+ for item in fetcher.items:
+ if item.status == item.STAT_ERROR:
+ print("Some error ocured: '%s'" % item.error_text)
+ pass
+ else:
+ self.handle_deb(item)
+
+ def clean_rootfs(self):
+ return
+ subprocess.run(["rm", "-rf", self.rootfs])
+
+ def apt_run(self, pkglist):
+ apt_pkg.init()
+
+ sources = apt_pkg.SourceList()
+ sources.read_main_list()
+
+ progress = apt.progress.text.AcquireProgress()
+
+ self.cache = apt_pkg.Cache()
+ self.cache.update(progress, sources)
+ self.cache = apt_pkg.Cache()
+
+ self.depcache = apt_pkg.DepCache(self.cache)
+
+ self.mark_essential()
+ #1 - required, 2 - important, 3 - standard, 4 - optional, 5 - extra
+ self.mark_by_prio(1)
+ self.mark_list(pkglist)
+
+ fetcher = apt_pkg.Acquire(progress)
+ pm = apt_pkg.PackageManager(self.depcache)
+
+ recs = apt_pkg.PackageRecords(self.cache)
+ pm.get_archives(fetcher, sources, recs)
+
+ self.handle_repo(fetcher)
+
+ self.clean_rootfs()
+
+def main():
+ workdir = "./rootfs"
+ repodir = "./apt"
+ repodbdir = "./db"
+ mirror="http://deb.debian.org/debian"
+ arch="amd64"
+ distro="debian"
+ codename="bullseye"
+ components="main,contrib,non-free"
+ keydir = "/etc/apt/trusted.gpg.d"
+ check_gpg = True
+
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "", [
+ "workdir=",
+ "repodir=",
+ "repodbdir=",
+ "mirror=",
+ "arch=",
+ "distro=",
+ "codename=",
+ "components=",
+ "keydir=",
+ "no-check-gpg",
+ ])
+ except getopt.GetoptError as msg:
+ print("Error: " + str(msg))
+ sys.exit(2)
+
+ for opt, arg in opts:
+ if opt in ("--workdir"):
+ workdir = arg
+ if opt in ("--repodir"):
+ repodir = arg
+ if opt in ("--repodbdir"):
+ repodbdir = arg
+ if opt in ("--mirror"):
+ mirror = arg
+ if opt in ("--arch"):
+ arch = arg
+ if opt in ("--distro"):
+ distro = arg
+ if opt in ("--codename"):
+ codename = arg
+ if opt in ("--components"):
+ components = arg
+ if opt in ("--keydir"):
+ keydir = arg
+ if opt in ("--no-check-gpg"):
+ check_gpg = False
+
+ preinst = args
+
+ debrepo = DebRepo(workdir, repodir, repodbdir, mirror, arch, distro, codename, components, keydir, check_gpg)
+
+ debrepo.create_rootfs()
+ debrepo.create_repo_dist()
+ debrepo.apt_config()
+ debrepo.apt_run(preinst)
+
+
+if __name__ == "__main__":
+ main()
--
2.20.1
^ permalink raw reply [flat|nested] 3+ messages in thread