* [PATCH v9 1/8] isar-bootstrap: Move common parts to bbclass
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 2/8] meta: Add mmdebstrap recipe Anton Mikanovich
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
To allow implementing other bootstrap recipes move some functionality
supposed to be reusable to bbclass.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/bootstrap.bbclass | 228 ++++++++++++++++++
.../isar-bootstrap/isar-bootstrap.inc | 221 +----------------
2 files changed, 230 insertions(+), 219 deletions(-)
create mode 100644 meta/classes/bootstrap.bbclass
diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass
new file mode 100644
index 00000000..b392677a
--- /dev/null
+++ b/meta/classes/bootstrap.bbclass
@@ -0,0 +1,228 @@
+# This software is a part of Isar.
+# Copyright (c) Siemens AG, 2018-2021
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+FILESEXTRAPATHS:append = ":${BBPATH}"
+
+SRC_URI = " \
+ file://locale \
+ file://chroot-setup.sh \
+ ${DISTRO_BOOTSTRAP_KEYS} \
+ ${THIRD_PARTY_APT_KEYS}"
+
+BOOTSTRAP_FOR_HOST ?= "0"
+
+APTPREFS = "${WORKDIR}/apt-preferences"
+APTSRCS = "${WORKDIR}/apt-sources"
+APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
+DISTRO_BOOTSTRAP_KEYFILES = ""
+THIRD_PARTY_APT_KEYFILES = ""
+DISTRO_BOOTSTRAP_KEYS ?= ""
+THIRD_PARTY_APT_KEYS ?= ""
+DEPLOY_ISAR_BOOTSTRAP ?= ""
+DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
+DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
+BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
+BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
+
+# reproducible builds, only enabled if ISAR_USE_APT_SNAPSHOT
+ISAR_APT_SNAPSHOT_MIRROR ??= ""
+APT_SNAPSHOT_DATE = "${@ get_apt_snapshot_date(d)}"
+
+python () {
+ distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
+ third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split()
+ topdir = d.getVar("TOPDIR")
+
+ # The cached repo key can be both for bootstrapping and apt package
+ # installation afterwards. However, bootstrap will include the key into
+ # the rootfs automatically thus the right place is distro_bootstrap_keys.
+
+ if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ own_pub_key = d.getVar("BASE_REPO_KEY")
+ if own_pub_key:
+ distro_bootstrap_keys += own_pub_key.split()
+
+ for key in distro_bootstrap_keys:
+ fetcher = bb.fetch2.Fetch([key], d)
+ filename = os.path.relpath(fetcher.localpath(key), topdir)
+ d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename)
+
+ for key in third_party_apt_keys:
+ fetcher = bb.fetch2.Fetch([key], d)
+ filename = os.path.relpath(fetcher.localpath(key), topdir)
+ d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
+
+ distro_apt_sources = get_aptsources_list(d)
+ for file in distro_apt_sources:
+ d.appendVar("SRC_URI", " file://%s" % file)
+
+ distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
+ for file in distro_apt_preferences.split():
+ d.appendVar("SRC_URI", " file://%s" % file)
+}
+
+def aggregate_files(d, file_list, file_out):
+ import shutil
+
+ with open(file_out, "wb") as out_fd:
+ for entry in file_list:
+ entry_real = bb.parse.resolve_file(entry, d)
+ with open(entry_real, "rb") as in_fd:
+ shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
+ out_fd.write("\n".encode())
+
+def parse_aptsources_list_line(source_list_line):
+ import re
+
+ s = source_list_line.strip()
+
+ if not s or s.startswith("#"):
+ return None
+
+ type, s = re.split("\s+", s, maxsplit=1)
+ if type not in ["deb", "deb-src"]:
+ return None
+
+ options = ""
+ options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s)
+ if options_match:
+ options = options_match.group(0).strip()
+ s = s[options_match.end():]
+
+ source, s = re.split("\s+", s, maxsplit=1)
+
+ if s.startswith("/"):
+ suite = ""
+ else:
+ suite, s = re.split("\s+", s, maxsplit=1)
+
+ components = " ".join(s.split())
+
+ return [type, options, source, suite, components]
+
+def get_apt_snapshot_date(d):
+ import time
+ source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP')
+ return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch)))
+
+def get_apt_source_mirror(d, aptsources_entry_list):
+ import re
+
+ # this is executed during parsing. No error checking possible
+ use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT'))
+ snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR')
+ if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
+ elif use_snapshot and snapshot_mirror:
+ premirrors = snapshot_mirror
+ else:
+ premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or ""
+ mirror_list = [entry.split()
+ for entry in premirrors.split('\\n')
+ if any(entry)]
+
+ for regex, replace in mirror_list:
+ match = re.search(regex, aptsources_entry_list[2])
+
+ if match:
+ new_aptsources_entry_list = aptsources_entry_list.copy()
+ new_aptsources_entry_list[2] = re.sub(regex, replace,
+ aptsources_entry_list[2],
+ count = 1)
+ if use_snapshot:
+ new_aptsources_entry_list[1] = "[check-valid-until=no]"
+ return new_aptsources_entry_list
+
+ return aptsources_entry_list
+
+def aggregate_aptsources_list(d, file_list, file_out):
+ import shutil
+
+ with open(file_out, "wb") as out_fd:
+ for entry in file_list:
+ entry_real = bb.parse.resolve_file(entry, d)
+ with open(entry_real, "r") as in_fd:
+ for line in in_fd:
+ parsed = parse_aptsources_list_line(line)
+ if parsed:
+ parsed = get_apt_source_mirror(d, parsed)
+ out_fd.write(" ".join(parsed).encode())
+ else:
+ out_fd.write(line.encode())
+ out_fd.write("\n".encode())
+ out_fd.write("\n".encode())
+
+def get_aptsources_list(d):
+ import errno
+ from collections import OrderedDict
+ apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
+ apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
+ for p in apt_sources_list:
+ try:
+ bb.parse.resolve_file(p, d)
+ except FileNotFoundError as e:
+ bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"')
+ return apt_sources_list
+
+def generate_distro_sources(d):
+ apt_sources_list = get_aptsources_list(d)
+ for entry in apt_sources_list:
+ with open(bb.parse.resolve_file(entry, d), "r") as in_fd:
+ for line in in_fd:
+ parsed = parse_aptsources_list_line(line)
+ if parsed:
+ parsed = get_apt_source_mirror(d, parsed)
+ yield parsed
+
+def get_distro_primary_source_entry(d):
+ for source in generate_distro_sources(d):
+ if source[0] == "deb":
+ return source[2:]
+ bb.fatal('Invalid apt sources list')
+
+def get_distro_source(d):
+ return get_distro_primary_source_entry(d)[0]
+
+def get_distro_suite(d):
+ return get_distro_primary_source_entry(d)[1]
+
+def get_distro_components_argument(d):
+ components = get_distro_primary_source_entry(d)[2]
+ if components and components.strip():
+ return "--components=" + ",".join(components.split())
+ else:
+ return ""
+
+do_apt_config_prepare[dirs] = "${WORKDIR}"
+do_apt_config_prepare[vardeps] += " \
+ APTPREFS \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \
+ DEBDISTRONAME \
+ APTSRCS \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
+ ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \
+ "
+python do_apt_config_prepare() {
+ apt_preferences_out = d.getVar("APTPREFS")
+ apt_preferences_list = (
+ d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
+
+ apt_sources_out = d.getVar("APTSRCS")
+ apt_sources_init_out = d.getVar("APTSRCS_INIT")
+ apt_sources_list = get_aptsources_list(d)
+
+ aggregate_files(d, apt_sources_list, apt_sources_init_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+}
+addtask apt_config_prepare before do_bootstrap after do_unpack
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+ rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
+}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index faf22a50..5c9eb5c8 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -5,193 +5,20 @@
#
# SPDX-License-Identifier: MIT
+inherit bootstrap
+
LICENSE = "gpl-2.0"
LIC_FILES_CHKSUM = "file://${LAYERDIR_core}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
-FILESPATH:prepend := "${THISDIR}/files:"
-SRC_URI = " \
- file://locale \
- file://chroot-setup.sh \
- ${DISTRO_BOOTSTRAP_KEYS} \
- ${THIRD_PARTY_APT_KEYS}"
PV = "1.0"
-BOOTSTRAP_FOR_HOST ?= "0"
-
DEBOOTSTRAP ?= "qemu-debootstrap"
ROOTFSDIR = "${WORKDIR}/rootfs"
-APTPREFS = "${WORKDIR}/apt-preferences"
-APTSRCS = "${WORKDIR}/apt-sources"
-APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-DISTRO_BOOTSTRAP_KEYFILES = ""
-THIRD_PARTY_APT_KEYFILES = ""
-DISTRO_BOOTSTRAP_KEYS ?= ""
-THIRD_PARTY_APT_KEYS ?= ""
-DEPLOY_ISAR_BOOTSTRAP ?= ""
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
-DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
-BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
-BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
-FILESEXTRAPATHS:append = ":${BBPATH}"
-# reproducible builds, only enabled if ISAR_USE_APT_SNAPSHOT
-ISAR_APT_SNAPSHOT_MIRROR ??= ""
-APT_SNAPSHOT_DATE = "${@ get_apt_snapshot_date(d)}"
inherit deb-dl-dir
-python () {
- distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
- third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split()
- topdir = d.getVar("TOPDIR")
-
- # The cached repo key can be both for bootstrapping and apt package
- # installation afterwards. However, debootstrap will include the key into
- # the rootfs automatically thus the right place is distro_bootstrap_keys.
-
- if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- own_pub_key = d.getVar("BASE_REPO_KEY")
- if own_pub_key:
- distro_bootstrap_keys += own_pub_key.split()
-
- for key in distro_bootstrap_keys:
- fetcher = bb.fetch2.Fetch([key], d)
- filename = os.path.relpath(fetcher.localpath(key), topdir)
- d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename)
-
- for key in third_party_apt_keys:
- fetcher = bb.fetch2.Fetch([key], d)
- filename = os.path.relpath(fetcher.localpath(key), topdir)
- d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
-
- distro_apt_sources = get_aptsources_list(d)
- for file in distro_apt_sources:
- d.appendVar("SRC_URI", " file://%s" % file)
-
- distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
- for file in distro_apt_preferences.split():
- d.appendVar("SRC_URI", " file://%s" % file)
-}
-
-def aggregate_files(d, file_list, file_out):
- import shutil
-
- with open(file_out, "wb") as out_fd:
- for entry in file_list:
- entry_real = bb.parse.resolve_file(entry, d)
- with open(entry_real, "rb") as in_fd:
- shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
- out_fd.write("\n".encode())
-
-def parse_aptsources_list_line(source_list_line):
- import re
-
- s = source_list_line.strip()
-
- if not s or s.startswith("#"):
- return None
-
- type, s = re.split("\s+", s, maxsplit=1)
- if type not in ["deb", "deb-src"]:
- return None
-
- options = ""
- options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s)
- if options_match:
- options = options_match.group(0).strip()
- s = s[options_match.end():]
-
- source, s = re.split("\s+", s, maxsplit=1)
-
- if s.startswith("/"):
- suite = ""
- else:
- suite, s = re.split("\s+", s, maxsplit=1)
-
- components = " ".join(s.split())
-
- return [type, options, source, suite, components]
-
-def get_apt_snapshot_date(d):
- import time
- source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP')
- return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch)))
-
-def get_apt_source_mirror(d, aptsources_entry_list):
- import re
-
- # this is executed during parsing. No error checking possible
- use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT'))
- snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR')
- if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
- elif use_snapshot and snapshot_mirror:
- premirrors = snapshot_mirror
- else:
- premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or ""
- mirror_list = [entry.split()
- for entry in premirrors.split('\\n')
- if any(entry)]
-
- for regex, replace in mirror_list:
- match = re.search(regex, aptsources_entry_list[2])
-
- if match:
- new_aptsources_entry_list = aptsources_entry_list.copy()
- new_aptsources_entry_list[2] = re.sub(regex, replace,
- aptsources_entry_list[2],
- count = 1)
- if use_snapshot:
- new_aptsources_entry_list[1] = "[check-valid-until=no]"
- return new_aptsources_entry_list
-
- return aptsources_entry_list
-
-def aggregate_aptsources_list(d, file_list, file_out):
- import shutil
-
- with open(file_out, "wb") as out_fd:
- for entry in file_list:
- entry_real = bb.parse.resolve_file(entry, d)
- with open(entry_real, "r") as in_fd:
- for line in in_fd:
- parsed = parse_aptsources_list_line(line)
- if parsed:
- parsed = get_apt_source_mirror(d, parsed)
- out_fd.write(" ".join(parsed).encode())
- else:
- out_fd.write(line.encode())
- out_fd.write("\n".encode())
- out_fd.write("\n".encode())
-
-def get_aptsources_list(d):
- import errno
- from collections import OrderedDict
- apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
- apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
- for p in apt_sources_list:
- try:
- bb.parse.resolve_file(p, d)
- except FileNotFoundError as e:
- bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"')
- return apt_sources_list
-
-def generate_distro_sources(d):
- apt_sources_list = get_aptsources_list(d)
- for entry in apt_sources_list:
- with open(bb.parse.resolve_file(entry, d), "r") as in_fd:
- for line in in_fd:
- parsed = parse_aptsources_list_line(line)
- if parsed:
- parsed = get_apt_source_mirror(d, parsed)
- yield parsed
-
-def get_distro_primary_source_entry(d):
- for source in generate_distro_sources(d):
- if source[0] == "deb":
- return source[2:]
- bb.fatal('Invalid apt sources list')
-
def get_distro_have_https_source(d):
return any(source[2].startswith("https://") for source in generate_distro_sources(d))
@@ -213,19 +40,6 @@ def get_distro_needs_gpg_support(d):
OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-def get_distro_source(d):
- return get_distro_primary_source_entry(d)[0]
-
-def get_distro_suite(d):
- return get_distro_primary_source_entry(d)[1]
-
-def get_distro_components_argument(d):
- components = get_distro_primary_source_entry(d)[2]
- if components and components.strip():
- return "--components=" + ",".join(components.split())
- else:
- return ""
-
APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
@@ -249,32 +63,6 @@ do_generate_keyrings() {
}
addtask generate_keyrings before do_build after do_unpack
-do_apt_config_prepare[dirs] = "${WORKDIR}"
-do_apt_config_prepare[vardeps] += " \
- APTPREFS \
- ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \
- DEBDISTRONAME \
- APTSRCS \
- ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
- DEPLOY_ISAR_BOOTSTRAP \
- ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \
- "
-python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS")
- apt_preferences_list = (
- d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
-
- apt_sources_out = d.getVar("APTSRCS")
- apt_sources_init_out = d.getVar("APTSRCS_INIT")
- apt_sources_list = get_aptsources_list(d)
-
- aggregate_files(d, apt_sources_list, apt_sources_init_out)
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
-}
-addtask apt_config_prepare before do_bootstrap after do_unpack
-
def get_host_release():
import platform
rel = platform.release()
@@ -446,8 +234,3 @@ python do_bootstrap_setscene() {
addtask do_bootstrap_setscene
do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
-}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 2/8] meta: Add mmdebstrap recipe
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 1/8] isar-bootstrap: Move common parts to bbclass Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 3/8] meta: Allow selecting bootstrap providers Anton Mikanovich
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
It can be used as debootstrap alternative for rootfs prepare.
Internally, it uses apt and allows to bootstrap the distro from
multiple repositories.
chroot-setup.sh and locale are copied from debootstrap recipe.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/bootstrap.bbclass | 1 +
.../isar-mmdebstrap/isar-mmdebstrap-host.bb | 17 ++
.../isar-mmdebstrap/isar-mmdebstrap-target.bb | 12 +
.../isar-mmdebstrap/isar-mmdebstrap.inc | 230 ++++++++++++++++++
4 files changed, 260 insertions(+)
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass
index b392677a..deb00b23 100644
--- a/meta/classes/bootstrap.bbclass
+++ b/meta/classes/bootstrap.bbclass
@@ -26,6 +26,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
+BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO_ARCH')}"
# reproducible builds, only enabled if ISAR_USE_APT_SNAPSHOT
ISAR_APT_SNAPSHOT_MIRROR ??= ""
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
new file mode 100644
index 00000000..66c8d11e
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
@@ -0,0 +1,17 @@
+# Minimal host Debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+Description = "Minimal host Debian root file system"
+
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+
+BOOTSTRAP_FOR_HOST = "1"
+
+require isar-mmdebstrap.inc
+
+HOST_DISTRO_BOOTSTRAP_KEYS ?= ""
+DISTRO_BOOTSTRAP_KEYS = "${HOST_DISTRO_BOOTSTRAP_KEYS}"
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
new file mode 100644
index 00000000..84a89ff1
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
@@ -0,0 +1,12 @@
+# Minimal target Debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+Description = "Minimal target Debian root file system"
+
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+
+require isar-mmdebstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
new file mode 100644
index 00000000..72096ae2
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -0,0 +1,230 @@
+# Minimal debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+inherit bootstrap
+inherit compat
+inherit deb-dl-dir
+
+FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
+
+ROOTFSDIR = "${WORKDIR}/rootfs"
+DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
+DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
+DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
+BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
+
+def get_distro_primary_source_entry(d):
+ for source in generate_distro_sources(d):
+ if source[0] == "deb":
+ return source[2:]
+ bb.fatal('Invalid apt sources list')
+
+def get_distro_have_https_source(d):
+ return any(source[2].startswith("https://") for source in generate_distro_sources(d))
+
+def get_distro_needs_https_support(d):
+ if get_distro_have_https_source(d):
+ return "https-support"
+ else:
+ return ""
+
+OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
+
+def get_distro_needs_gpg_support(d):
+ if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
+ d.getVar("THIRD_PARTY_APT_KEYS") or \
+ d.getVar("BASE_REPO_KEY"):
+ return "gnupg"
+ else:
+ return ""
+
+OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
+
+APT_KEYS_DIR = "${WORKDIR}/aptkeys"
+DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
+
+do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
+do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
+do_generate_keyrings[network] = "${TASK_USE_SUDO}"
+do_generate_keyrings() {
+ if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
+ chmod 777 "${APT_KEYS_DIR}"
+ for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
+ cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ done
+ fi
+ if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
+ for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
+ sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
+ cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ done
+ fi
+}
+addtask generate_keyrings before do_build after do_unpack
+
+do_bootstrap[vardeps] += " \
+ DISTRO_APT_PREMIRRORS \
+ ISAR_ENABLE_COMPAT_ARCH \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
+ "
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
+do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+
+do_bootstrap() {
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
+ bbfatal "${DISTRO_ARCH} does not have a compat arch"
+ fi
+ fi
+ bootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
+ if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
+ bootstrap_args="$bootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ fi
+ E="${@ isar_export_proxies(d)}"
+ export BOOTSTRAP_FOR_HOST
+
+ deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${ROOTFSDIR}"
+ mkdir -p "${ROOTFSDIR}"
+
+ arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+
+ sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
+ $arch_param \
+ --mode=unshare \
+ ${@get_distro_components_argument(d)} \
+ "${@get_distro_suite(d)}" \
+ "${WORKDIR}/rootfs.tar.zst" \
+ "${@get_distro_source(d)}"
+
+ sudo -E -s <<'EOSUDO'
+ set -e
+
+ tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # Install apt config
+ mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+ install -v -m644 "${APTPREFS}" \
+ "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+ mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
+ mkdir -p ${ROOTFSDIR}/base-apt
+ mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
+ else
+ install -v -m644 "${APTSRCS}" \
+ "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+ fi
+ install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
+ rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+ rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
+ find ${APT_KEYS_DIR}/ -type f | while read keyfile
+ do
+ MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
+ echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
+ export GNUPGHOME="${MY_GPGHOME}"
+ APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
+
+ kfn="$(basename $keyfile)"
+ cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
+ chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
+ --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
+ rm "${ROOTFSDIR}/tmp/$kfn"
+
+ echo "Removing ${MY_GPGHOME}"
+ rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
+ done
+
+ # Set locale
+ install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
+
+ sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
+ chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
+
+ # setup chroot
+ install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
+ "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+
+ # update APT
+ mount -o bind,private /dev ${ROOTFSDIR}/dev
+ mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
+ mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
+ mount -t proc none ${ROOTFSDIR}/proc
+ mount --rbind /sys ${ROOTFSDIR}/sys
+ mount --make-rslave ${ROOTFSDIR}/sys
+
+ export DEBIAN_FRONTEND=noninteractive
+
+ if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
+ chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
+ fi
+
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
+ fi
+
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
+ -o APT::Update::Error-Mode=any
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+ -o Debug::pkgProblemResolver=yes
+
+ umount -l "${ROOTFSDIR}/dev/shm"
+ umount -l "${ROOTFSDIR}/dev/pts"
+ umount -l "${ROOTFSDIR}/dev"
+ umount -l "${ROOTFSDIR}/proc"
+ umount -l "${ROOTFSDIR}/sys"
+ umount -l "${ROOTFSDIR}/base-apt" || true
+
+ # Finalize bootstrap by setting the link in deploy
+ ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ # Cleanup apt cache
+ sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+}
+addtask bootstrap before do_build after do_generate_keyrings
+
+SSTATETASKS += "do_bootstrap"
+SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
+SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
+
+bootstrap_sstate_prepare() {
+ # this runs in SSTATE_BUILDDIR, which will be deleted automatically
+ sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
+}
+
+bootstrap_sstate_finalize() {
+ # this runs in SSTATE_INSTDIR
+ # we should restore symlinks after using tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
+ sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
+ fi
+}
+
+python do_bootstrap_setscene() {
+ sstate_setscene(d)
+}
+
+addtask do_bootstrap_setscene
+do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 3/8] meta: Allow selecting bootstrap providers
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 1/8] isar-bootstrap: Move common parts to bbclass Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 2/8] meta: Add mmdebstrap recipe Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 4/8] testsuite: Allow variable " Anton Mikanovich
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Both isar-bootstrap or isar-mmdebstrap can be used for initial
rootfs preparation.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta-isar/conf/local.conf.sample | 4 ++++
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 3 +++
meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb | 2 ++
meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb | 2 ++
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb | 2 ++
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb | 2 ++
7 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 17455015..a7862352 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -221,3 +221,7 @@ USER_isar[flags] += "clear-text-password"
# To know more details about this variable and how to set the value refer below
# https://reproducible-builds.org/docs/source-date-epoch/
#SOURCE_DATE_EPOCH =
+
+# Uncomment this to use old isar-bootstrap provider for rootfs prepare
+#PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
+#PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 498fbfd6..f684e969 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -183,7 +183,7 @@ rootfs_install_pkgs_install() {
do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
do_rootfs_install[vardepsexclude] += "IMAGE_ROOTFS"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
+do_rootfs_install[depends] = "bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
do_rootfs_install[recrdeptask] = "do_deploy_deb"
do_rootfs_install[network] = "${TASK_USE_SUDO}"
python do_rootfs_install() {
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 4cfa8b10..faaeede3 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -193,3 +193,6 @@ include conf/machine/${MACHINE}.conf
include conf/distro/${DISTRO}.conf
PATCHRESOLVE ?= "noop"
+
+PREFERRED_PROVIDER_bootstrap-host ??= "isar-mmdebstrap-host"
+PREFERRED_PROVIDER_bootstrap-target ??= "isar-mmdebstrap-target"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
index 4f90fd01..12798488 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
@@ -9,6 +9,8 @@ Description = "Minimal host Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-host"
+
BOOTSTRAP_FOR_HOST = "1"
require isar-bootstrap.inc
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
index c66cb3b3..69e97b33 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
@@ -9,4 +9,6 @@ Description = "Minimal target Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-target"
+
require isar-bootstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
index 66c8d11e..6de9c21a 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
@@ -9,6 +9,8 @@ Description = "Minimal host Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-host"
+
BOOTSTRAP_FOR_HOST = "1"
require isar-mmdebstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
index 84a89ff1..227ff04c 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
@@ -9,4 +9,6 @@ Description = "Minimal target Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-target"
+
require isar-mmdebstrap.inc
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 4/8] testsuite: Allow variable bootstrap providers
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
` (2 preceding siblings ...)
2024-06-14 11:34 ` [PATCH v9 3/8] meta: Allow selecting bootstrap providers Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 5/8] mmdebstrap: Fix missing dpkg available Anton Mikanovich
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
If we can use different recipes provide bootstrap, recipe name should
be obtained from target environment.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
testsuite/cibase.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index 0f6997af..c6117b36 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -179,8 +179,9 @@ class CIBaseTest(CIBuilder):
# Rebuild image
self.move_in_build_dir('tmp', 'tmp_before_sstate')
self.bitbake(image_target, **kwargs)
+ bootstrap_target_task = self.getVars('PREFERRED_PROVIDER_bootstrap-target', target=image_target)
if not all([
- check_executed_tasks('isar-bootstrap-target',
+ check_executed_tasks(bootstrap_target_task,
['do_bootstrap_setscene', '!do_bootstrap']),
check_executed_tasks('sbuild-chroot-target',
['do_rootfs_install_setscene', '!do_rootfs_install']),
@@ -204,7 +205,7 @@ class CIBaseTest(CIBuilder):
self.move_in_build_dir('tmp', 'tmp_middle_sstate')
self.bitbake(package_target, **kwargs)
if not all([
- check_executed_tasks('isar-bootstrap-target',
+ check_executed_tasks(bootstrap_target_task,
['do_bootstrap_setscene']),
check_executed_tasks('sbuild-chroot-target',
['!do_sbuildchroot_deploy']),
@@ -218,7 +219,7 @@ class CIBaseTest(CIBuilder):
process.run(f'find {self.build_dir}/sstate-cache/ -name sstate:hello:* -delete')
self.bitbake(image_target, **kwargs)
if not all([
- check_executed_tasks('isar-bootstrap-target',
+ check_executed_tasks(bootstrap_target_task,
['do_bootstrap_setscene', '!do_bootstrap']),
check_executed_tasks('sbuild-chroot-target',
['do_rootfs_install_setscene', '!do_rootfs_install']),
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 5/8] mmdebstrap: Fix missing dpkg available
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
` (3 preceding siblings ...)
2024-06-14 11:34 ` [PATCH v9 4/8] testsuite: Allow variable " Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 6/8] mmdebstrap: Move preparations to hooks Anton Mikanovich
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
In some distros /var/lib/dpkg/available is required for
do_rootfs_install task.
The fix for it is present in jessie-or-older mmdebstrap hook in
Bookworm and later, but maybe-jessie-or-older wrapper do not handle
distro version check correctly.
That's why we need to check and apply hook manually or create
/var/lib/dpkg/available by hand for older distros.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 72096ae2..4f56a0b8 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -17,6 +17,14 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
+# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
+MM_HOOK_JESSIE = "/usr/share/mmdebstrap/hooks/jessie-or-older"
+DPKG_HOOKS ?= "${@'--hook-dir='+d.getVar('MM_HOOK_JESSIE') \
+ if os.path.isdir(d.getVar('MM_HOOK_JESSIE')) \
+ else '--customize-hook=\'touch /var/lib/dpkg/available\''}"
+MMHOOKS:ubuntu-focal ?= "${DPKG_HOOKS}"
+MMHOOKS:debian-buster ?= "${DPKG_HOOKS}"
+
def get_distro_primary_source_entry(d):
for source in generate_distro_sources(d):
if source[0] == "deb":
@@ -98,6 +106,7 @@ do_bootstrap() {
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
$arch_param \
--mode=unshare \
+ ${MMHOOKS} \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 6/8] mmdebstrap: Move preparations to hooks
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
` (4 preceding siblings ...)
2024-06-14 11:34 ` [PATCH v9 5/8] mmdebstrap: Fix missing dpkg available Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 7/8] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 8/8] user_manual.md: Update boostrap related documentation Anton Mikanovich
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
Also migrate from apt-key to gpg.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-mmdebstrap/isar-mmdebstrap.inc | 237 +++++++++---------
1 file changed, 113 insertions(+), 124 deletions(-)
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 4f56a0b8..c41dd3a1 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -13,8 +13,6 @@ FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
ROOTFSDIR = "${WORKDIR}/rootfs"
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
@@ -25,53 +23,29 @@ DPKG_HOOKS ?= "${@'--hook-dir='+d.getVar('MM_HOOK_JESSIE') \
MMHOOKS:ubuntu-focal ?= "${DPKG_HOOKS}"
MMHOOKS:debian-buster ?= "${DPKG_HOOKS}"
-def get_distro_primary_source_entry(d):
- for source in generate_distro_sources(d):
- if source[0] == "deb":
- return source[2:]
- bb.fatal('Invalid apt sources list')
-
-def get_distro_have_https_source(d):
- return any(source[2].startswith("https://") for source in generate_distro_sources(d))
-
-def get_distro_needs_https_support(d):
- if get_distro_have_https_source(d):
- return "https-support"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
-
-def get_distro_needs_gpg_support(d):
- if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
- d.getVar("THIRD_PARTY_APT_KEYS") or \
- d.getVar("BASE_REPO_KEY"):
- return "gnupg"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-
-APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
-do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
-do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
+do_generate_keyrings[dirs] = "${DEBDIR}"
do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
do_generate_keyrings[network] = "${TASK_USE_SUDO}"
do_generate_keyrings() {
+ export GNUPGHOME="$(mktemp -td gpghomeXXXXXXXXXX)"
if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
- chmod 777 "${APT_KEYS_DIR}"
for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ gpg --no-autostart --keyring "gnupg-ring:${DISTRO_BOOTSTRAP_KEYRING}" \
+ --no-default-keyring --import $keyfile
done
fi
if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
- sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ gpg --no-autostart --keyring "gnupg-ring:${DISTRO_BOOTSTRAP_KEYRING}" \
+ --no-default-keyring --import $keyfile
done
fi
+ if [ -r "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
+ chmod o+r "${DISTRO_BOOTSTRAP_KEYRING}"
+ fi
}
addtask generate_keyrings before do_build after do_unpack
@@ -80,10 +54,12 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -93,121 +69,134 @@ do_bootstrap() {
bootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
bootstrap_args="$bootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST
- deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${ROOTFSDIR}"
mkdir -p "${ROOTFSDIR}"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
+ bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
+
+ # no need to sync /var/cache/apt/archives if base-apt used
+ syncin='echo skip sync-in'
+ syncout='echo skip sync-out'
+ extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
+ extra_extract="$syncout"
+ # save mmdebstrap tempdir for cleanup
+ extra_essential="mkdir -p \$1/$base_apt_tmp && \
+ echo \$1 > ${WORKDIR}/mmtmpdir && \
+ mount --bind '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
+ # replace base-apt mount in tmp with /base-apt mount
+ extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
+ \$1/etc/apt/sources.list.d/*.list && \
+ mkdir -p \$1/base-apt && \
+ mount --bind '${REPO_BASE_DIR}' \$1/base-apt && \
+ chroot \$1 apt-get update -y \
+ -o APT::Update::Error-Mode=any && \
+ umount \$1/base-apt && \
+ umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
+ umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
+ else
+ deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
+ install -v -m644 "${APTSRCS}" \
+ "${WORKDIR}/sources.list.d/bootstrap.list"
+
+ syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
+ "$1/var/cache/apt/archives/" || true'
+ syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "$1/var/cache/apt/archives/"*.deb \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
+ extra_setup="$syncin"
+ extra_extract="$syncout"
+ # prefetch apt debs because mmdebstrap will clean them on next stage
+ extra_essential='apt-get install apt -y -d \
+ -o Dir::State="$1/var/lib/apt" \
+ -o Dir::Etc="$1/etc/apt" \
+ -o Dir::Cache="$1/var/cache/apt" \
+ -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
+ extra_essential="$extra_essential && $syncout"
+ extra_customize="$syncout"
+ fi
+
+ if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
+ export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+ fi
+
arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
+ fi
+
+ # Cleanup mounts if fails
+ trap 'exit 1' INT HUP QUIT TERM ALRM USR1
+ trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
+ && rm "${WORKDIR}/mmtmpdir"; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
+ && sudo umount $tmpdir/$base_apt_tmp; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
+ && sudo umount $tmpdir/base-apt; \
+ [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
+ [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
+ && sudo umount $base_apt_tmp \
+ && rm -rf --one-file-system $base_apt_tmp' EXIT
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
$arch_param \
--mode=unshare \
${MMHOOKS} \
+ --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
+ --setup-hook="$extra_setup" \
+ --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
+ --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
+ --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
+ --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
+ --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
+ --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
+ --extract-hook="$extra_extract" \
+ --essential-hook="$extra_essential" \
+ --customize-hook="$extra_customize" \
+ --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
+ --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
+ --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
+ --skip=cleanup/apt \
+ --skip=download/empty \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
- "${@get_distro_source(d)}"
+ "$bootstrap_list"
sudo -E -s <<'EOSUDO'
set -e
tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
- # Install apt config
- mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
-
- mkdir -p ${ROOTFSDIR}/base-apt
- mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- else
- install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- fi
- install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
- rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
- find ${APT_KEYS_DIR}/ -type f | while read keyfile
- do
- MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
- echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
- export GNUPGHOME="${MY_GPGHOME}"
- APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
-
- kfn="$(basename $keyfile)"
- cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
- chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
- --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
- rm "${ROOTFSDIR}/tmp/$kfn"
-
- echo "Removing ${MY_GPGHOME}"
- rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
- done
-
- # Set locale
- install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
-
- sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-
# setup chroot
install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
"${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
- # update APT
- mount -o bind,private /dev ${ROOTFSDIR}/dev
- mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
- mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
- mount -t proc none ${ROOTFSDIR}/proc
- mount --rbind /sys ${ROOTFSDIR}/sys
- mount --make-rslave ${ROOTFSDIR}/sys
-
- export DEBIAN_FRONTEND=noninteractive
-
- if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
- fi
-
- if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
- fi
-
- chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
- -o APT::Update::Error-Mode=any
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
- chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
- -o Debug::pkgProblemResolver=yes
-
- umount -l "${ROOTFSDIR}/dev/shm"
- umount -l "${ROOTFSDIR}/dev/pts"
- umount -l "${ROOTFSDIR}/dev"
- umount -l "${ROOTFSDIR}/proc"
- umount -l "${ROOTFSDIR}/sys"
- umount -l "${ROOTFSDIR}/base-apt" || true
-
# Finalize bootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
- deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
-
- # Cleanup apt cache
- sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
+ deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
+ fi
}
addtask bootstrap before do_build after do_generate_keyrings
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 7/8] isar-bootstrap: Use tar output instead of directory
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
` (5 preceding siblings ...)
2024-06-14 11:34 ` [PATCH v9 6/8] mmdebstrap: Move preparations to hooks Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
2024-06-14 11:34 ` [PATCH v9 8/8] user_manual.md: Update boostrap related documentation Anton Mikanovich
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Do not unpack tar with rootfs content inside bootstrap recipe.
This change can allow to remove sudo usage during the bootstrap later.
Using tar output requires moving chroot prepare step to the later
unpack step.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/rootfs.bbclass | 11 +++++----
.../isar-bootstrap/isar-bootstrap.inc | 22 +++++++++---------
.../isar-mmdebstrap/isar-mmdebstrap.inc | 23 ++++---------------
3 files changed, 23 insertions(+), 33 deletions(-)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index f684e969..a479b37a 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -73,12 +73,15 @@ rootfs_do_qemu() {
fi
}
-BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
-BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
+BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
rootfs_prepare[weight] = "25"
rootfs_prepare(){
- sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+ sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # setup chroot
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
}
ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -227,7 +230,7 @@ cache_deb_src() {
# Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
# relies on that through out the build. Copy that state information instead of apt-get update
# which generates a new state from upstream.
- sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+ sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5c9eb5c8..efec42c6 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -162,7 +162,6 @@ do_bootstrap() {
# setup chroot
install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
# update APT
mount -o bind,private /dev ${ROOTFSDIR}/dev
@@ -195,8 +194,11 @@ do_bootstrap() {
umount -l "${ROOTFSDIR}/sys"
umount -l "${ROOTFSDIR}/base-apt" || true
+ # Compress rootfs for compatibility
+ tar --zstd -cf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" .
+
# Finalize debootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+ ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
EOSUDO
deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
@@ -212,19 +214,17 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- lopts="--one-file-system --exclude=var/cache/apt/archives"
- sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
- sudo chown $(id -u):$(id -g) bootstrap.tar
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
- # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
- # - after restoring from cache, there will be a tar which we unpack and then delete
- if [ -f bootstrap.tar ]; then
- sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
- sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
- rm bootstrap.tar
+ # we should restore symlinks after using tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index c41dd3a1..f1ad891b 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -11,7 +11,6 @@ inherit deb-dl-dir
FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
-ROOTFSDIR = "${WORKDIR}/rootfs"
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
@@ -73,9 +72,6 @@ do_bootstrap() {
fi
E="${@ isar_export_proxies(d)}"
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
- mkdir -p "${ROOTFSDIR}"
-
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
@@ -181,18 +177,9 @@ do_bootstrap() {
"${WORKDIR}/rootfs.tar.zst" \
"$bootstrap_list"
- sudo -E -s <<'EOSUDO'
- set -e
-
- tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
-
- # setup chroot
- install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+ # Finalize bootstrap by setting the link in deploy
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
- # Finalize bootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
-EOSUDO
if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
@@ -206,7 +193,7 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
@@ -214,8 +201,8 @@ bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
# we should restore symlinks after using tar
if [ -f bootstrap.tar.zst ]; then
- mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
- sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
"${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v9 8/8] user_manual.md: Update boostrap related documentation
2024-06-14 11:34 [PATCH v9 0/8] Migrate to mmdebstrap Anton Mikanovich
` (6 preceding siblings ...)
2024-06-14 11:34 ` [PATCH v9 7/8] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
@ 2024-06-14 11:34 ` Anton Mikanovich
7 siblings, 0 replies; 9+ messages in thread
From: Anton Mikanovich @ 2024-06-14 11:34 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Update documentation in user manual and recipes API changelog.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
RECIPE-API-CHANGELOG.md | 13 +++++++++++++
doc/user_manual.md | 15 ++++++++++++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index 12ea93ec..589dae66 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -629,3 +629,16 @@ into kernel kbuild package.
Only the "host" specific package is built automatically at cross builds.
* Support emulated module build with cross-compiled kernel for linux-module
+
+### Default boostrap recipe changed to mmdebstrap
+
+New virtual packages bootstrap-host and bootstrap-target are introduced.
+There are two providers of bootstrap-host/-target currently:
+ * isar-mmdebstrap: deafult one using mmdebstrap to prepare rootfs
+ * isar-bootstrap: previous bootstrap implementation left for compatibility
+
+So default bootstrap procedure is now performing with mmdebstrap.
+Previous implementation still can be selected by setting in local.conf:
+
+PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
+PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
diff --git a/doc/user_manual.md b/doc/user_manual.md
index b12c7692..d3b41ae8 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -66,7 +66,8 @@ The steps below describe how to build the images provided by default.
### Install Host Tools
-The supported host system is >= buster.
+The supported host system is >= buster for debootstrap and >= bullseye for
+default mmdebstrap provider.
Building `debian-trixie` requires host system >= bookworm.
@@ -75,7 +76,9 @@ Install the following packages:
apt install \
binfmt-support \
bzip2 \
- debootstrap \
+ mmdebstrap \
+ arch-test \
+ apt-utils \
dpkg-dev \
gettext-base \
git \
@@ -93,6 +96,12 @@ apt install \
zstd
```
+If using isar-bootstrap provider, debootstrap should be installed instead of
+mmdebstrap:
+```
+apt install debootstrap
+```
+
If your host is >= buster, also install the following package.
```
apt install python3-distutils
@@ -135,7 +144,7 @@ apt install qemu
### Setup Sudo
-Isar requires `sudo` rights without password to work with `chroot` and `debootstrap`. To add them, use the following steps:
+Isar requires `sudo` rights without password to work with `chroot`. To add them, use the following steps:
```
# visudo
```
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread