* [PATCH v11 1/8] isar-bootstrap: Move common parts to bbclass
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 2/8] meta: Add mmdebstrap recipe Anton Mikanovich
` (9 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
To allow implementing other bootstrap recipes move some functionality
supposed to be reusable to bbclass.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/bootstrap.bbclass | 225 ++++++++++++++++++
.../isar-bootstrap/isar-bootstrap.inc | 219 +----------------
2 files changed, 227 insertions(+), 217 deletions(-)
create mode 100644 meta/classes/bootstrap.bbclass
diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass
new file mode 100644
index 00000000..870465a9
--- /dev/null
+++ b/meta/classes/bootstrap.bbclass
@@ -0,0 +1,225 @@
+# This software is a part of Isar.
+# Copyright (c) Siemens AG, 2018-2021
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+FILESEXTRAPATHS:append = ":${BBPATH}"
+
+SRC_URI = " \
+ file://locale \
+ file://chroot-setup.sh \
+ ${DISTRO_BOOTSTRAP_KEYS} \
+ ${THIRD_PARTY_APT_KEYS}"
+
+BOOTSTRAP_FOR_HOST ?= "0"
+
+APTPREFS = "${WORKDIR}/apt-preferences"
+APTSRCS = "${WORKDIR}/apt-sources"
+APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
+DISTRO_BOOTSTRAP_KEYFILES = ""
+THIRD_PARTY_APT_KEYFILES = ""
+DISTRO_BOOTSTRAP_KEYS ?= ""
+THIRD_PARTY_APT_KEYS ?= ""
+DEPLOY_ISAR_BOOTSTRAP ?= ""
+DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
+DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
+BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
+BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
+ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}"
+
+python () {
+ distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
+ third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split()
+ topdir = d.getVar("TOPDIR")
+
+ # The cached repo key can be both for bootstrapping and apt package
+ # installation afterwards. However, bootstrap will include the key into
+ # the rootfs automatically thus the right place is distro_bootstrap_keys.
+
+ if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ own_pub_key = d.getVar("BASE_REPO_KEY")
+ if own_pub_key:
+ distro_bootstrap_keys += own_pub_key.split()
+
+ for key in distro_bootstrap_keys:
+ fetcher = bb.fetch2.Fetch([key], d)
+ filename = os.path.relpath(fetcher.localpath(key), topdir)
+ d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename)
+
+ for key in third_party_apt_keys:
+ fetcher = bb.fetch2.Fetch([key], d)
+ filename = os.path.relpath(fetcher.localpath(key), topdir)
+ d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
+
+ distro_apt_sources = get_aptsources_list(d)
+ for file in distro_apt_sources:
+ d.appendVar("SRC_URI", " file://%s" % file)
+
+ distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
+ for file in distro_apt_preferences.split():
+ d.appendVar("SRC_URI", " file://%s" % file)
+}
+
+def aggregate_files(d, file_list, file_out):
+ import shutil
+
+ with open(file_out, "wb") as out_fd:
+ for entry in file_list:
+ entry_real = bb.parse.resolve_file(entry, d)
+ with open(entry_real, "rb") as in_fd:
+ shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
+ out_fd.write("\n".encode())
+
+def parse_aptsources_list_line(source_list_line):
+ import re
+
+ s = source_list_line.strip()
+
+ if not s or s.startswith("#"):
+ return None
+
+ type, s = re.split("\s+", s, maxsplit=1)
+ if type not in ["deb", "deb-src"]:
+ return None
+
+ options = ""
+ options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s)
+ if options_match:
+ options = options_match.group(0).strip()
+ s = s[options_match.end():]
+
+ source, s = re.split("\s+", s, maxsplit=1)
+
+ if s.startswith("/"):
+ suite = ""
+ else:
+ suite, s = re.split("\s+", s, maxsplit=1)
+
+ components = " ".join(s.split())
+
+ return [type, options, source, suite, components]
+
+def get_isar_apt_snapshot_date(d):
+ import time
+ source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP')
+ return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch)))
+
+def get_apt_source_mirror(d, aptsources_entry_list):
+ import re
+
+ # this is executed during parsing. No error checking possible
+ use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT'))
+ snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR')
+ if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
+ elif use_snapshot and snapshot_mirror:
+ premirrors = snapshot_mirror
+ else:
+ premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or ""
+ mirror_list = [entry.split()
+ for entry in premirrors.split('\\n')
+ if any(entry)]
+
+ for regex, replace in mirror_list:
+ match = re.search(regex, aptsources_entry_list[2])
+
+ if match:
+ new_aptsources_entry_list = aptsources_entry_list.copy()
+ new_aptsources_entry_list[2] = re.sub(regex, replace,
+ aptsources_entry_list[2],
+ count = 1)
+ if use_snapshot:
+ new_aptsources_entry_list[1] = "[check-valid-until=no]"
+ return new_aptsources_entry_list
+
+ return aptsources_entry_list
+
+def aggregate_aptsources_list(d, file_list, file_out):
+ import shutil
+
+ with open(file_out, "wb") as out_fd:
+ for entry in file_list:
+ entry_real = bb.parse.resolve_file(entry, d)
+ with open(entry_real, "r") as in_fd:
+ for line in in_fd:
+ parsed = parse_aptsources_list_line(line)
+ if parsed:
+ parsed = get_apt_source_mirror(d, parsed)
+ out_fd.write(" ".join(parsed).encode())
+ else:
+ out_fd.write(line.encode())
+ out_fd.write("\n".encode())
+ out_fd.write("\n".encode())
+
+def get_aptsources_list(d):
+ import errno
+ from collections import OrderedDict
+ apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
+ apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
+ for p in apt_sources_list:
+ try:
+ bb.parse.resolve_file(p, d)
+ except FileNotFoundError as e:
+ bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"')
+ return apt_sources_list
+
+def generate_distro_sources(d):
+ apt_sources_list = get_aptsources_list(d)
+ for entry in apt_sources_list:
+ with open(bb.parse.resolve_file(entry, d), "r") as in_fd:
+ for line in in_fd:
+ parsed = parse_aptsources_list_line(line)
+ if parsed:
+ parsed = get_apt_source_mirror(d, parsed)
+ yield parsed
+
+def get_distro_primary_source_entry(d):
+ for source in generate_distro_sources(d):
+ if source[0] == "deb":
+ return source[2:]
+ bb.fatal('Invalid apt sources list')
+
+def get_distro_source(d):
+ return get_distro_primary_source_entry(d)[0]
+
+def get_distro_suite(d):
+ return get_distro_primary_source_entry(d)[1]
+
+def get_distro_components_argument(d):
+ components = get_distro_primary_source_entry(d)[2]
+ if components and components.strip():
+ return "--components=" + ",".join(components.split())
+ else:
+ return ""
+
+do_apt_config_prepare[dirs] = "${WORKDIR}"
+do_apt_config_prepare[vardeps] += " \
+ APTPREFS \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \
+ DEBDISTRONAME \
+ APTSRCS \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
+ ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \
+ "
+python do_apt_config_prepare() {
+ apt_preferences_out = d.getVar("APTPREFS")
+ apt_preferences_list = (
+ d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
+
+ apt_sources_out = d.getVar("APTSRCS")
+ apt_sources_init_out = d.getVar("APTSRCS_INIT")
+ apt_sources_list = get_aptsources_list(d)
+
+ aggregate_files(d, apt_sources_list, apt_sources_init_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+}
+addtask apt_config_prepare before do_bootstrap after do_unpack
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+ rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
+}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index e155aade..0e5ab57d 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -5,191 +5,20 @@
#
# SPDX-License-Identifier: MIT
+inherit bootstrap
+
LICENSE = "gpl-2.0"
LIC_FILES_CHKSUM = "file://${LAYERDIR_core}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
-FILESPATH:prepend := "${THISDIR}/files:"
-SRC_URI = " \
- file://locale \
- file://chroot-setup.sh \
- ${DISTRO_BOOTSTRAP_KEYS} \
- ${THIRD_PARTY_APT_KEYS}"
PV = "1.0"
-BOOTSTRAP_FOR_HOST ?= "0"
-
DEBOOTSTRAP ?= "qemu-debootstrap"
ROOTFSDIR = "${WORKDIR}/rootfs"
-APTPREFS = "${WORKDIR}/apt-preferences"
-APTSRCS = "${WORKDIR}/apt-sources"
-APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-DISTRO_BOOTSTRAP_KEYFILES = ""
-THIRD_PARTY_APT_KEYFILES = ""
-DISTRO_BOOTSTRAP_KEYS ?= ""
-THIRD_PARTY_APT_KEYS ?= ""
-DEPLOY_ISAR_BOOTSTRAP ?= ""
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
-DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
-BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
-BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
-FILESEXTRAPATHS:append = ":${BBPATH}"
-ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}"
inherit deb-dl-dir
-python () {
- distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
- third_party_apt_keys = (d.getVar("THIRD_PARTY_APT_KEYS") or "").split()
- topdir = d.getVar("TOPDIR")
-
- # The cached repo key can be both for bootstrapping and apt package
- # installation afterwards. However, debootstrap will include the key into
- # the rootfs automatically thus the right place is distro_bootstrap_keys.
-
- if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- own_pub_key = d.getVar("BASE_REPO_KEY")
- if own_pub_key:
- distro_bootstrap_keys += own_pub_key.split()
-
- for key in distro_bootstrap_keys:
- fetcher = bb.fetch2.Fetch([key], d)
- filename = os.path.relpath(fetcher.localpath(key), topdir)
- d.appendVar("DISTRO_BOOTSTRAP_KEYFILES", " ${TOPDIR}/%s" % filename)
-
- for key in third_party_apt_keys:
- fetcher = bb.fetch2.Fetch([key], d)
- filename = os.path.relpath(fetcher.localpath(key), topdir)
- d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
-
- distro_apt_sources = get_aptsources_list(d)
- for file in distro_apt_sources:
- d.appendVar("SRC_URI", " file://%s" % file)
-
- distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
- for file in distro_apt_preferences.split():
- d.appendVar("SRC_URI", " file://%s" % file)
-}
-
-def aggregate_files(d, file_list, file_out):
- import shutil
-
- with open(file_out, "wb") as out_fd:
- for entry in file_list:
- entry_real = bb.parse.resolve_file(entry, d)
- with open(entry_real, "rb") as in_fd:
- shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
- out_fd.write("\n".encode())
-
-def parse_aptsources_list_line(source_list_line):
- import re
-
- s = source_list_line.strip()
-
- if not s or s.startswith("#"):
- return None
-
- type, s = re.split("\s+", s, maxsplit=1)
- if type not in ["deb", "deb-src"]:
- return None
-
- options = ""
- options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s)
- if options_match:
- options = options_match.group(0).strip()
- s = s[options_match.end():]
-
- source, s = re.split("\s+", s, maxsplit=1)
-
- if s.startswith("/"):
- suite = ""
- else:
- suite, s = re.split("\s+", s, maxsplit=1)
-
- components = " ".join(s.split())
-
- return [type, options, source, suite, components]
-
-def get_isar_apt_snapshot_date(d):
- import time
- source_date_epoch = d.getVar('ISAR_APT_SNAPSHOT_TIMESTAMP')
- return time.strftime('%Y%m%dT%H%M%SZ', time.gmtime(int(source_date_epoch)))
-
-def get_apt_source_mirror(d, aptsources_entry_list):
- import re
-
- # this is executed during parsing. No error checking possible
- use_snapshot = bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT'))
- snapshot_mirror = d.getVar('DISTRO_APT_SNAPSHOT_PREMIRROR')
- if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
- elif use_snapshot and snapshot_mirror:
- premirrors = snapshot_mirror
- else:
- premirrors = d.getVar('DISTRO_APT_PREMIRRORS') or ""
- mirror_list = [entry.split()
- for entry in premirrors.split('\\n')
- if any(entry)]
-
- for regex, replace in mirror_list:
- match = re.search(regex, aptsources_entry_list[2])
-
- if match:
- new_aptsources_entry_list = aptsources_entry_list.copy()
- new_aptsources_entry_list[2] = re.sub(regex, replace,
- aptsources_entry_list[2],
- count = 1)
- if use_snapshot:
- new_aptsources_entry_list[1] = "[check-valid-until=no]"
- return new_aptsources_entry_list
-
- return aptsources_entry_list
-
-def aggregate_aptsources_list(d, file_list, file_out):
- import shutil
-
- with open(file_out, "wb") as out_fd:
- for entry in file_list:
- entry_real = bb.parse.resolve_file(entry, d)
- with open(entry_real, "r") as in_fd:
- for line in in_fd:
- parsed = parse_aptsources_list_line(line)
- if parsed:
- parsed = get_apt_source_mirror(d, parsed)
- out_fd.write(" ".join(parsed).encode())
- else:
- out_fd.write(line.encode())
- out_fd.write("\n".encode())
- out_fd.write("\n".encode())
-
-def get_aptsources_list(d):
- import errno
- from collections import OrderedDict
- apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
- apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
- for p in apt_sources_list:
- try:
- bb.parse.resolve_file(p, d)
- except FileNotFoundError as e:
- bb.fatal(os.strerror(errno.ENOENT) + ' "' + p + '"')
- return apt_sources_list
-
-def generate_distro_sources(d):
- apt_sources_list = get_aptsources_list(d)
- for entry in apt_sources_list:
- with open(bb.parse.resolve_file(entry, d), "r") as in_fd:
- for line in in_fd:
- parsed = parse_aptsources_list_line(line)
- if parsed:
- parsed = get_apt_source_mirror(d, parsed)
- yield parsed
-
-def get_distro_primary_source_entry(d):
- for source in generate_distro_sources(d):
- if source[0] == "deb":
- return source[2:]
- bb.fatal('Invalid apt sources list')
-
def get_distro_have_https_source(d):
return any(source[2].startswith("https://") for source in generate_distro_sources(d))
@@ -211,19 +40,6 @@ def get_distro_needs_gpg_support(d):
OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-def get_distro_source(d):
- return get_distro_primary_source_entry(d)[0]
-
-def get_distro_suite(d):
- return get_distro_primary_source_entry(d)[1]
-
-def get_distro_components_argument(d):
- components = get_distro_primary_source_entry(d)[2]
- if components and components.strip():
- return "--components=" + ",".join(components.split())
- else:
- return ""
-
APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
@@ -247,32 +63,6 @@ do_generate_keyrings() {
}
addtask generate_keyrings before do_build after do_unpack
-do_apt_config_prepare[dirs] = "${WORKDIR}"
-do_apt_config_prepare[vardeps] += " \
- APTPREFS \
- ${DISTRO_VARS_PREFIX}DISTRO_APT_PREFERENCES \
- DEBDISTRONAME \
- APTSRCS \
- ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
- DEPLOY_ISAR_BOOTSTRAP \
- ${@'DISTRO_APT_SNAPSHOT_PREMIRROR' if bb.utils.to_boolean(d.getVar('ISAR_USE_APT_SNAPSHOT')) else ''} \
- "
-python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS")
- apt_preferences_list = (
- d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
-
- apt_sources_out = d.getVar("APTSRCS")
- apt_sources_init_out = d.getVar("APTSRCS_INIT")
- apt_sources_list = get_aptsources_list(d)
-
- aggregate_files(d, apt_sources_list, apt_sources_init_out)
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
-}
-addtask apt_config_prepare before do_bootstrap after do_unpack
-
def get_host_release():
import platform
rel = platform.release()
@@ -448,8 +238,3 @@ python do_bootstrap_setscene() {
addtask do_bootstrap_setscene
do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
-}
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-2-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v11 2/8] meta: Add mmdebstrap recipe
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 1/8] isar-bootstrap: Move common parts to bbclass Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-07 6:24 ` 'MOESSBAUER, Felix' via isar-users
2024-11-06 8:21 ` [PATCH v11 3/8] meta: Allow selecting bootstrap providers Anton Mikanovich
` (8 subsequent siblings)
10 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
It can be used as debootstrap alternative for rootfs prepare.
Internally, it uses apt and allows to bootstrap the distro from
multiple repositories.
chroot-setup.sh and locale are copied from debootstrap recipe.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/bootstrap.bbclass | 1 +
.../isar-mmdebstrap/isar-mmdebstrap-host.bb | 17 ++
.../isar-mmdebstrap/isar-mmdebstrap-target.bb | 12 +
.../isar-mmdebstrap/isar-mmdebstrap.inc | 235 ++++++++++++++++++
4 files changed, 265 insertions(+)
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass
index 870465a9..f5b92808 100644
--- a/meta/classes/bootstrap.bbclass
+++ b/meta/classes/bootstrap.bbclass
@@ -26,6 +26,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
DISTRO_VARS_PREFIX ?= "${@'HOST_' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'BASE_DISTRO')}"
+BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO_ARCH')}"
ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}"
python () {
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
new file mode 100644
index 00000000..66c8d11e
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
@@ -0,0 +1,17 @@
+# Minimal host Debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+Description = "Minimal host Debian root file system"
+
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+
+BOOTSTRAP_FOR_HOST = "1"
+
+require isar-mmdebstrap.inc
+
+HOST_DISTRO_BOOTSTRAP_KEYS ?= ""
+DISTRO_BOOTSTRAP_KEYS = "${HOST_DISTRO_BOOTSTRAP_KEYS}"
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
new file mode 100644
index 00000000..84a89ff1
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
@@ -0,0 +1,12 @@
+# Minimal target Debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+Description = "Minimal target Debian root file system"
+
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+
+require isar-mmdebstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
new file mode 100644
index 00000000..658d45be
--- /dev/null
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -0,0 +1,235 @@
+# Minimal debian root file system
+#
+# This software is a part of Isar.
+# Copyright (C) 2024 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+inherit bootstrap
+inherit compat
+inherit deb-dl-dir
+
+FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
+
+ROOTFSDIR = "${WORKDIR}/rootfs"
+DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
+DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
+DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
+BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
+
+def get_distro_primary_source_entry(d):
+ for source in generate_distro_sources(d):
+ if source[0] == "deb":
+ return source[2:]
+ bb.fatal('Invalid apt sources list')
+
+def get_distro_have_https_source(d):
+ return any(source[2].startswith("https://") for source in generate_distro_sources(d))
+
+def get_distro_needs_https_support(d):
+ if get_distro_have_https_source(d):
+ return "https-support"
+ else:
+ return ""
+
+OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
+
+def get_distro_needs_gpg_support(d):
+ if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
+ d.getVar("THIRD_PARTY_APT_KEYS") or \
+ d.getVar("BASE_REPO_KEY"):
+ return "gnupg"
+ else:
+ return ""
+
+OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
+
+APT_KEYS_DIR = "${WORKDIR}/aptkeys"
+DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
+
+do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
+do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
+do_generate_keyrings[network] = "${TASK_USE_SUDO}"
+do_generate_keyrings() {
+ if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
+ chmod 777 "${APT_KEYS_DIR}"
+ for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
+ cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ done
+ fi
+ if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
+ for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
+ sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
+ cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ done
+ fi
+}
+addtask generate_keyrings before do_build after do_unpack
+
+do_bootstrap[vardeps] += " \
+ DISTRO_APT_PREMIRRORS \
+ ISAR_ENABLE_COMPAT_ARCH \
+ ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
+ "
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
+do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+
+do_bootstrap() {
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
+ bbfatal "${DISTRO_ARCH} does not have a compat arch"
+ fi
+ fi
+ bootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
+ if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
+ bootstrap_args="$bootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ fi
+ E="${@ isar_export_proxies(d)}"
+ export BOOTSTRAP_FOR_HOST
+
+ deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${ROOTFSDIR}"
+ mkdir -p "${ROOTFSDIR}"
+
+ arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+
+ sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
+ $arch_param \
+ --mode=unshare \
+ ${@get_distro_components_argument(d)} \
+ "${@get_distro_suite(d)}" \
+ "${WORKDIR}/rootfs.tar.zst" \
+ "${@get_distro_source(d)}"
+
+ sudo -E -s <<'EOSUDO'
+ set -e
+
+ tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # Install apt config
+ mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+ install -v -m644 "${APTPREFS}" \
+ "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+ mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
+ mkdir -p ${ROOTFSDIR}/base-apt
+ mount -o bind,private ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
+ else
+ install -v -m644 "${APTSRCS}" \
+ "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+ fi
+ install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
+ rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+ rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
+ find ${APT_KEYS_DIR}/ -type f | while read keyfile
+ do
+ MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
+ echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
+ export GNUPGHOME="${MY_GPGHOME}"
+ APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
+
+ kfn="$(basename $keyfile)"
+ cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
+ chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
+ --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
+ rm "${ROOTFSDIR}/tmp/$kfn"
+
+ echo "Removing ${MY_GPGHOME}"
+ rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
+ done
+
+ # Set locale
+ install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
+
+ sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
+ chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
+
+ # update APT
+ mount -o bind,private /dev ${ROOTFSDIR}/dev
+ mount -o bind,private /dev/pts ${ROOTFSDIR}/dev/pts
+ mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
+ mount -t proc none ${ROOTFSDIR}/proc
+ mount -o bind,private /sys ${ROOTFSDIR}/sys
+ mount --make-rslave ${ROOTFSDIR}/sys
+
+ export DEBIAN_FRONTEND=noninteractive
+
+ if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
+ chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
+ fi
+
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
+ fi
+
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
+ -o APT::Update::Error-Mode=any
+
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
+
+ # setup chroot
+ install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
+ "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
+ chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+ -o Debug::pkgProblemResolver=yes
+
+ umount "${ROOTFSDIR}/dev/shm"
+ umount "${ROOTFSDIR}/dev/pts"
+ umount "${ROOTFSDIR}/dev"
+ umount "${ROOTFSDIR}/proc"
+ umount "${ROOTFSDIR}/sys"
+ if mountpoint -q "${ROOTFSDIR}/base-apt"; then
+ umount "${ROOTFSDIR}/base-apt"
+ fi
+
+ # Finalize bootstrap by setting the link in deploy
+ ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ # Cleanup apt cache
+ sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+}
+addtask bootstrap before do_build after do_generate_keyrings
+
+SSTATETASKS += "do_bootstrap"
+SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
+SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
+
+bootstrap_sstate_prepare() {
+ # this runs in SSTATE_BUILDDIR, which will be deleted automatically
+ sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
+}
+
+bootstrap_sstate_finalize() {
+ # this runs in SSTATE_INSTDIR
+ # we should restore symlinks after using tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
+ sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
+ fi
+}
+
+python do_bootstrap_setscene() {
+ sstate_setscene(d)
+}
+
+addtask do_bootstrap_setscene
+do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-3-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 2/8] meta: Add mmdebstrap recipe
2024-11-06 8:21 ` [PATCH v11 2/8] meta: Add mmdebstrap recipe Anton Mikanovich
@ 2024-11-07 6:24 ` 'MOESSBAUER, Felix' via isar-users
2024-11-12 16:06 ` Anton Mikanovich
0 siblings, 1 reply; 15+ messages in thread
From: 'MOESSBAUER, Felix' via isar-users @ 2024-11-07 6:24 UTC (permalink / raw)
To: amikan, isar-users
On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
> It can be used as debootstrap alternative for rootfs prepare.
>
> Internally, it uses apt and allows to bootstrap the distro from
> multiple repositories.
>
> chroot-setup.sh and locale are copied from debootstrap recipe.
>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> meta/classes/bootstrap.bbclass | 1 +
> .../isar-mmdebstrap/isar-mmdebstrap-host.bb | 17 ++
> .../isar-mmdebstrap/isar-mmdebstrap-target.bb | 12 +
> .../isar-mmdebstrap/isar-mmdebstrap.inc | 235
> ++++++++++++++++++
> 4 files changed, 265 insertions(+)
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-host.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-target.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap.inc
>
> diff --git a/meta/classes/bootstrap.bbclass
> b/meta/classes/bootstrap.bbclass
> index 870465a9..f5b92808 100644
> --- a/meta/classes/bootstrap.bbclass
> +++ b/meta/classes/bootstrap.bbclass
> @@ -26,6 +26,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
> DISTRO_VARS_PREFIX ?= "${@'HOST_' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
> BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
> BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else
> 'BASE_DISTRO')}"
> +BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else
> 'DISTRO_ARCH')}"
> ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}"
>
> python () {
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
> new file mode 100644
> index 00000000..66c8d11e
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
> @@ -0,0 +1,17 @@
> +# Minimal host Debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +Description = "Minimal host Debian root file system"
> +
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-
> host_${DISTRO}-${DISTRO_ARCH}"
> +
> +BOOTSTRAP_FOR_HOST = "1"
> +
> +require isar-mmdebstrap.inc
> +
> +HOST_DISTRO_BOOTSTRAP_KEYS ?= ""
> +DISTRO_BOOTSTRAP_KEYS = "${HOST_DISTRO_BOOTSTRAP_KEYS}"
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> target.bb
> new file mode 100644
> index 00000000..84a89ff1
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
> @@ -0,0 +1,12 @@
> +# Minimal target Debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +Description = "Minimal target Debian root file system"
> +
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-
> ${DISTRO_ARCH}"
> +
> +require isar-mmdebstrap.inc
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> new file mode 100644
> index 00000000..658d45be
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -0,0 +1,235 @@
> +# Minimal debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +inherit bootstrap
> +inherit compat
> +inherit deb-dl-dir
> +
> +FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-
> bootstrap/files"
> +
> +ROOTFSDIR = "${WORKDIR}/rootfs"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-
> certificates"
> +BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
> +
> +def get_distro_primary_source_entry(d):
> + for source in generate_distro_sources(d):
> + if source[0] == "deb":
> + return source[2:]
> + bb.fatal('Invalid apt sources list')
> +
> +def get_distro_have_https_source(d):
> + return any(source[2].startswith("https://") for source in
> generate_distro_sources(d))
> +
> +def get_distro_needs_https_support(d):
> + if get_distro_have_https_source(d):
> + return "https-support"
> + else:
> + return ""
> +
> +OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
> +
> +def get_distro_needs_gpg_support(d):
> + if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
> + d.getVar("THIRD_PARTY_APT_KEYS") or \
> + d.getVar("BASE_REPO_KEY"):
> + return "gnupg"
> + else:
> + return ""
> +
> +OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
> +
> +APT_KEYS_DIR = "${WORKDIR}/aptkeys"
> +DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
> +
> +do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
> +do_generate_keyrings[dirs] = "${DL_DIR}"
> +do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS
> THIRD_PARTY_APT_KEYS"
> +do_generate_keyrings[network] = "${TASK_USE_SUDO}"
> +do_generate_keyrings() {
> + if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
> + chmod 777 "${APT_KEYS_DIR}"
> + for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
> + cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> + done
> + fi
> + if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ];
> then
> + for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
> + sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add
> $keyfile
> + cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> + done
> + fi
> +}
> +addtask generate_keyrings before do_build after do_unpack
> +
> +do_bootstrap[vardeps] += " \
> + DISTRO_APT_PREMIRRORS \
> + ISAR_ENABLE_COMPAT_ARCH \
> + ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> + "
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> +do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> +do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
> +
> +do_bootstrap() {
> + if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> + if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
> + bbfatal "${DISTRO_ARCH} does not have a compat arch"
> + fi
> + fi
> + bootstrap_args="--verbose --variant=minbase --
> include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
> + if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> + bootstrap_args="$bootstrap_args --
> keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> + fi
> + E="${@ isar_export_proxies(d)}"
> + export BOOTSTRAP_FOR_HOST
> +
> + deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> + sudo rm -rf --one-file-system "${ROOTFSDIR}"
> + mkdir -p "${ROOTFSDIR}"
> +
> + arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> +
> + sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
Hi, why does this need to run as root? mmdebstrap with the unshare
backend can also run unprivileged.
> + $arch_param \
> + --mode=unshare \
> + ${@get_distro_components_argument(d)} \
> + "${@get_distro_suite(d)}" \
> + "${WORKDIR}/rootfs.tar.zst" \
> + "${@get_distro_source(d)}"
The apt options are missing. This is especially relevant when running
against the snapshot mirrors. In debootstrap this was not possible, but
as mmdepstrap uses the host apt, options can be passed (with --aptopt
<file>).
Felix
> +
> + sudo -E -s <<'EOSUDO'
> + set -e
> +
> + tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --
> exclude="./dev/console"
> +
> + # Install apt config
> + mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> + install -v -m644 "${APTPREFS}" \
> +
> "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> + mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> + if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> + line="file:///base-
> apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb ${line}" >
> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> + line="file:///base-
> apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb-src ${line}" >>
> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> +
> + mkdir -p ${ROOTFSDIR}/base-apt
> + mount -o bind,private ${REPO_BASE_DIR}
> ${ROOTFSDIR}/base-apt
> + else
> + install -v -m644 "${APTSRCS}" \
> +
> "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> + fi
> + install -v -m644 "${APTSRCS_INIT}"
> "${ROOTFSDIR}/etc/apt/sources-list"
> + rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> + rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
> + find ${APT_KEYS_DIR}/ -type f | while read keyfile
> + do
> + MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d
> /tmp/gpghomeXXXXXXXXXX)"
> + echo "Created temporary directory ${MY_GPGHOME} for gpg-
> agent"
> + export GNUPGHOME="${MY_GPGHOME}"
> + APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
> +
> + kfn="$(basename $keyfile)"
> + cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
> + chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon --
> /usr/bin/apt-key \
> + --keyring ${THIRD_PARTY_APT_KEYRING}
> ${APT_KEY_APPEND} add "/tmp/$kfn"
> + rm "${ROOTFSDIR}/tmp/$kfn"
> +
> + echo "Removing ${MY_GPGHOME}"
> + rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
> + done
> +
> + # Set locale
> + install -v -m644 "${WORKDIR}/locale"
> "${ROOTFSDIR}/etc/locale"
> +
> + sed -i '/en_US.UTF-8 UTF-8/s/^#//g'
> "${ROOTFSDIR}/etc/locale.gen"
> + chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> +
> + # update APT
> + mount -o bind,private /dev ${ROOTFSDIR}/dev
> + mount -o bind,private /dev/pts ${ROOTFSDIR}/dev/pts
> + mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
> + mount -t proc none ${ROOTFSDIR}/proc
> + mount -o bind,private /sys ${ROOTFSDIR}/sys
> + mount --make-rslave ${ROOTFSDIR}/sys
> +
> + export DEBIAN_FRONTEND=noninteractive
> +
> + if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
> + chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
> ${DISTRO_ARCH}
> + fi
> +
> + if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> + chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
> ${COMPAT_DISTRO_ARCH}
> + fi
> +
> + chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
> + -o APT::Update::Error-Mode=any
> +
> + chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
> +
> + # setup chroot
> + install -v -m755 "${WORKDIR}/chroot-setup.sh"
> "${ROOTFSDIR}/chroot-setup.sh"
> + "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> +
> + chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
> + chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> + -o Debug::pkgProblemResolver=yes
> +
> + umount "${ROOTFSDIR}/dev/shm"
> + umount "${ROOTFSDIR}/dev/pts"
> + umount "${ROOTFSDIR}/dev"
> + umount "${ROOTFSDIR}/proc"
> + umount "${ROOTFSDIR}/sys"
> + if mountpoint -q "${ROOTFSDIR}/base-apt"; then
> + umount "${ROOTFSDIR}/base-apt"
> + fi
> +
> + # Finalize bootstrap by setting the link in deploy
> + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> +EOSUDO
> + deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> +
> + # Cleanup apt cache
> + sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
> +}
> +addtask bootstrap before do_build after do_generate_keyrings
> +
> +SSTATETASKS += "do_bootstrap"
> +SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
> +SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> +
> +bootstrap_sstate_prepare() {
> + # this runs in SSTATE_BUILDDIR, which will be deleted
> automatically
> + sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> ./bootstrap.tar.zst
> + sudo chown $(id -u):$(id -g) bootstrap.tar.zst
> +}
> +
> +bootstrap_sstate_finalize() {
> + # this runs in SSTATE_INSTDIR
> + # we should restore symlinks after using tar
> + if [ -f bootstrap.tar.zst ]; then
> + mv bootstrap.tar.zst "$(dirname
> "${ROOTFSDIR}")/rootfs.tar.zst"
> + sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> + "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
> + fi
> +}
> +
> +python do_bootstrap_setscene() {
> + sstate_setscene(d)
> +}
> +
> +addtask do_bootstrap_setscene
> +do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> --
> 2.34.1
>
--
Siemens AG, Technology
Linux Expert Center
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/1853b41ea5fb9b099b1147bb5c65e9a0188d1985.camel%40siemens.com.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 2/8] meta: Add mmdebstrap recipe
2024-11-07 6:24 ` 'MOESSBAUER, Felix' via isar-users
@ 2024-11-12 16:06 ` Anton Mikanovich
0 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-12 16:06 UTC (permalink / raw)
To: MOESSBAUER, Felix, isar-users
07/11/2024 08:24, MOESSBAUER, Felix wrote:
> On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
>> +
>> + deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
>> ${BASE_DISTRO_CODENAME}"
>> + sudo rm -rf --one-file-system "${ROOTFSDIR}"
>> + mkdir -p "${ROOTFSDIR}"
>> +
>> + arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
>> +
>> + sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
> Hi, why does this need to run as root? mmdebstrap with the unshare
> backend can also run unprivileged.
Removing sudo here will be the next step (addressed in other patchset after
merging mmdebstrap) because this requires few more changes to make it work
(like removing sudo from deb_dl_dir_import/export). There is also uidmap
package missing in the latest kas container for this step.
>> + $arch_param \
>> + --mode=unshare \
>> + ${@get_distro_components_argument(d)} \
>> + "${@get_distro_suite(d)}" \
>> + "${WORKDIR}/rootfs.tar.zst" \
>> + "${@get_distro_source(d)}"
> The apt options are missing. This is especially relevant when running
> against the snapshot mirrors. In debootstrap this was not possible, but
> as mmdepstrap uses the host apt, options can be passed (with --aptopt
> <file>).
>
> Felix
Thanks for pointing on it, but I would like to keep the same
functionality like
debootstrap implementation had in this first splitting patchset, so will
keep
it for the separate patch.
>> +
>> + sudo -E -s <<'EOSUDO'
>> + set -e
>> +
>> + tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --
>> exclude="./dev/console"
>> +
>> + # Install apt config
>> + mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
>> + install -v -m644 "${APTPREFS}" \
>> +
>> "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
>> + mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
>> + if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>> + line="file:///base-
>> apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
>> + if [ -z "${BASE_REPO_KEY}" ]; then
>> + line="[trusted=yes] ${line}"
>> + fi
>> + echo "deb ${line}" >
>> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
>> + line="file:///base-
>> apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
>> + if [ -z "${BASE_REPO_KEY}" ]; then
>> + line="[trusted=yes] ${line}"
>> + fi
>> + echo "deb-src ${line}" >>
>> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
>> +
>> + mkdir -p ${ROOTFSDIR}/base-apt
>> + mount -o bind,private ${REPO_BASE_DIR}
>> ${ROOTFSDIR}/base-apt
>> + else
>> + install -v -m644 "${APTSRCS}" \
>> +
>> "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
>> + fi
>> + install -v -m644 "${APTSRCS_INIT}"
>> "${ROOTFSDIR}/etc/apt/sources-list"
>> + rm -f "${ROOTFSDIR}/etc/apt/sources.list"
>> + rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>> + find ${APT_KEYS_DIR}/ -type f | while read keyfile
>> + do
>> + MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d
>> /tmp/gpghomeXXXXXXXXXX)"
>> + echo "Created temporary directory ${MY_GPGHOME} for gpg-
>> agent"
>> + export GNUPGHOME="${MY_GPGHOME}"
>> + APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
>> +
>> + kfn="$(basename $keyfile)"
>> + cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
>> + chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon --
>> /usr/bin/apt-key \
>> + --keyring ${THIRD_PARTY_APT_KEYRING}
>> ${APT_KEY_APPEND} add "/tmp/$kfn"
>> + rm "${ROOTFSDIR}/tmp/$kfn"
>> +
>> + echo "Removing ${MY_GPGHOME}"
>> + rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
>> + done
>> +
>> + # Set locale
>> + install -v -m644 "${WORKDIR}/locale"
>> "${ROOTFSDIR}/etc/locale"
>> +
>> + sed -i '/en_US.UTF-8 UTF-8/s/^#//g'
>> "${ROOTFSDIR}/etc/locale.gen"
>> + chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
>> +
>> + # update APT
>> + mount -o bind,private /dev ${ROOTFSDIR}/dev
>> + mount -o bind,private /dev/pts ${ROOTFSDIR}/dev/pts
>> + mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
>> + mount -t proc none ${ROOTFSDIR}/proc
>> + mount -o bind,private /sys ${ROOTFSDIR}/sys
>> + mount --make-rslave ${ROOTFSDIR}/sys
>> +
>> + export DEBIAN_FRONTEND=noninteractive
>> +
>> + if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
>> ${DISTRO_ARCH}
>> + fi
>> +
>> + if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
>> ${COMPAT_DISTRO_ARCH}
>> + fi
>> +
>> + chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
>> + -o APT::Update::Error-Mode=any
>> +
>> + chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
>> +
>> + # setup chroot
>> + install -v -m755 "${WORKDIR}/chroot-setup.sh"
>> "${ROOTFSDIR}/chroot-setup.sh"
>> + "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>> +
>> + chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
>> + chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>> + -o Debug::pkgProblemResolver=yes
>> +
>> + umount "${ROOTFSDIR}/dev/shm"
>> + umount "${ROOTFSDIR}/dev/pts"
>> + umount "${ROOTFSDIR}/dev"
>> + umount "${ROOTFSDIR}/proc"
>> + umount "${ROOTFSDIR}/sys"
>> + if mountpoint -q "${ROOTFSDIR}/base-apt"; then
>> + umount "${ROOTFSDIR}/base-apt"
>> + fi
>> +
>> + # Finalize bootstrap by setting the link in deploy
>> + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>> +EOSUDO
>> + deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
>> ${BASE_DISTRO_CODENAME}"
>> +
>> + # Cleanup apt cache
>> + sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
>> +}
>> +addtask bootstrap before do_build after do_generate_keyrings
>> +
>> +SSTATETASKS += "do_bootstrap"
>> +SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
>> +SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>> +
>> +bootstrap_sstate_prepare() {
>> + # this runs in SSTATE_BUILDDIR, which will be deleted
>> automatically
>> + sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
>> ./bootstrap.tar.zst
>> + sudo chown $(id -u):$(id -g) bootstrap.tar.zst
>> +}
>> +
>> +bootstrap_sstate_finalize() {
>> + # this runs in SSTATE_INSTDIR
>> + # we should restore symlinks after using tar
>> + if [ -f bootstrap.tar.zst ]; then
>> + mv bootstrap.tar.zst "$(dirname
>> "${ROOTFSDIR}")/rootfs.tar.zst"
>> + sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
>> + "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>> + fi
>> +}
>> +
>> +python do_bootstrap_setscene() {
>> + sstate_setscene(d)
>> +}
>> +
>> +addtask do_bootstrap_setscene
>> +do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
>> --
>> 2.34.1
>>
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/e2853ca1-9cf9-49d5-8230-eb4d53118f3b%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v11 3/8] meta: Allow selecting bootstrap providers
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 1/8] isar-bootstrap: Move common parts to bbclass Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 2/8] meta: Add mmdebstrap recipe Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 4/8] testsuite: Allow variable " Anton Mikanovich
` (7 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Both isar-bootstrap or isar-mmdebstrap can be used for initial
rootfs preparation.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta-isar/conf/local.conf.sample | 4 ++++
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 3 +++
meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb | 2 ++
meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb | 2 ++
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb | 2 ++
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb | 2 ++
7 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 17455015..a7862352 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -221,3 +221,7 @@ USER_isar[flags] += "clear-text-password"
# To know more details about this variable and how to set the value refer below
# https://reproducible-builds.org/docs/source-date-epoch/
#SOURCE_DATE_EPOCH =
+
+# Uncomment this to use old isar-bootstrap provider for rootfs prepare
+#PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
+#PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index c7011508..e339d24f 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -227,7 +227,7 @@ rootfs_install_pkgs_install() {
do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
do_rootfs_install[vardepsexclude] += "IMAGE_ROOTFS"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
+do_rootfs_install[depends] = "bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
do_rootfs_install[recrdeptask] = "do_deploy_deb"
do_rootfs_install[network] = "${TASK_USE_SUDO}"
python do_rootfs_install() {
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 13966a62..289c90b7 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -195,3 +195,6 @@ include conf/machine/${MACHINE}.conf
include conf/distro/${DISTRO}.conf
PATCHRESOLVE ?= "noop"
+
+PREFERRED_PROVIDER_bootstrap-host ??= "isar-mmdebstrap-host"
+PREFERRED_PROVIDER_bootstrap-target ??= "isar-mmdebstrap-target"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
index 4f90fd01..12798488 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
@@ -9,6 +9,8 @@ Description = "Minimal host Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-host"
+
BOOTSTRAP_FOR_HOST = "1"
require isar-bootstrap.inc
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
index c66cb3b3..69e97b33 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
@@ -9,4 +9,6 @@ Description = "Minimal target Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-target"
+
require isar-bootstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
index 66c8d11e..6de9c21a 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
@@ -9,6 +9,8 @@ Description = "Minimal host Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-host"
+
BOOTSTRAP_FOR_HOST = "1"
require isar-mmdebstrap.inc
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
index 84a89ff1..227ff04c 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
@@ -9,4 +9,6 @@ Description = "Minimal target Debian root file system"
DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-${DISTRO_ARCH}"
+PROVIDES += "bootstrap-target"
+
require isar-mmdebstrap.inc
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-4-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v11 4/8] testsuite: Allow variable bootstrap providers
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (2 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 3/8] meta: Allow selecting bootstrap providers Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 5/8] mmdebstrap: Fix missing dpkg available Anton Mikanovich
` (6 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
If we can use different recipes provide bootstrap, recipe name should
be obtained from target environment.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
testsuite/cibase.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index cccac86c..075535b1 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -216,10 +216,13 @@ class CIBaseTest(CIBuilder):
# Rebuild image
self.move_in_build_dir('tmp', 'tmp_before_sstate')
self.bitbake(image_target, **kwargs)
+ bootstrap_target_task = CIUtils.getVars(
+ 'PREFERRED_PROVIDER_bootstrap-target', target=image_target
+ )
if not all(
[
check_executed_tasks(
- 'isar-bootstrap-target',
+ bootstrap_target_task,
['do_bootstrap_setscene', '!do_bootstrap'],
),
check_executed_tasks(
@@ -257,7 +260,7 @@ class CIBaseTest(CIBuilder):
if not all(
[
check_executed_tasks(
- 'isar-bootstrap-target', ['do_bootstrap_setscene']
+ bootstrap_target_task, ['do_bootstrap_setscene']
),
check_executed_tasks(
'sbuild-chroot-target', ['!do_sbuildchroot_deploy']
@@ -282,7 +285,7 @@ class CIBaseTest(CIBuilder):
if not all(
[
check_executed_tasks(
- 'isar-bootstrap-target',
+ bootstrap_target_task,
['do_bootstrap_setscene', '!do_bootstrap'],
),
check_executed_tasks(
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-5-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v11 5/8] mmdebstrap: Fix missing dpkg available
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (3 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 4/8] testsuite: Allow variable " Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-20 22:18 ` 'Jan Kiszka' via isar-users
2024-11-06 8:21 ` [PATCH v11 6/8] mmdebstrap: Move preparations to hooks Anton Mikanovich
` (5 subsequent siblings)
10 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
In some distros /var/lib/dpkg/available is required for
do_rootfs_install task.
The fix for it is present in jessie-or-older mmdebstrap hook in
Bookworm and later, but maybe-jessie-or-older wrapper do not handle
distro version check correctly.
That's why we need to check and apply hook manually or create
/var/lib/dpkg/available by hand for older distros.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 658d45be..e1c3832d 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -17,6 +17,14 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
+# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
+MM_HOOK_JESSIE = "/usr/share/mmdebstrap/hooks/jessie-or-older"
+DPKG_HOOKS ?= "${@'--hook-dir='+d.getVar('MM_HOOK_JESSIE') \
+ if os.path.isdir(d.getVar('MM_HOOK_JESSIE')) \
+ else '--customize-hook=\'touch /var/lib/dpkg/available\''}"
+MMHOOKS:ubuntu-focal ?= "${DPKG_HOOKS}"
+MMHOOKS:debian-buster ?= "${DPKG_HOOKS}"
+
def get_distro_primary_source_entry(d):
for source in generate_distro_sources(d):
if source[0] == "deb":
@@ -98,6 +106,7 @@ do_bootstrap() {
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
$arch_param \
--mode=unshare \
+ ${MMHOOKS} \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-6-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 5/8] mmdebstrap: Fix missing dpkg available
2024-11-06 8:21 ` [PATCH v11 5/8] mmdebstrap: Fix missing dpkg available Anton Mikanovich
@ 2024-11-20 22:18 ` 'Jan Kiszka' via isar-users
0 siblings, 0 replies; 15+ messages in thread
From: 'Jan Kiszka' via isar-users @ 2024-11-20 22:18 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
On 06.11.24 09:21, Anton Mikanovich wrote:
> In some distros /var/lib/dpkg/available is required for
> do_rootfs_install task.
> The fix for it is present in jessie-or-older mmdebstrap hook in
> Bookworm and later, but maybe-jessie-or-older wrapper do not handle
> distro version check correctly.
> That's why we need to check and apply hook manually or create
> /var/lib/dpkg/available by hand for older distros.
>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 9 +++++++++
> 1 file changed, 9 insertions(+)
>
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> index 658d45be..e1c3832d 100644
> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -17,6 +17,14 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
> DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
> BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
>
> +# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
> +MM_HOOK_JESSIE = "/usr/share/mmdebstrap/hooks/jessie-or-older"
> +DPKG_HOOKS ?= "${@'--hook-dir='+d.getVar('MM_HOOK_JESSIE') \
> + if os.path.isdir(d.getVar('MM_HOOK_JESSIE')) \
> + else '--customize-hook=\'touch /var/lib/dpkg/available\''}"
> +MMHOOKS:ubuntu-focal ?= "${DPKG_HOOKS}"
> +MMHOOKS:debian-buster ?= "${DPKG_HOOKS}"
This breaks downstream buster (or focal) when setting custom DISTRO. Fix
will follow.
Jan
> +
> def get_distro_primary_source_entry(d):
> for source in generate_distro_sources(d):
> if source[0] == "deb":
> @@ -98,6 +106,7 @@ do_bootstrap() {
> sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
> $arch_param \
> --mode=unshare \
> + ${MMHOOKS} \
> ${@get_distro_components_argument(d)} \
> "${@get_distro_suite(d)}" \
> "${WORKDIR}/rootfs.tar.zst" \
--
Siemens AG, Technology
Linux Expert Center
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/a26f9dfd-bd74-49bb-b295-78f04de1fc0e%40siemens.com.
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v11 6/8] mmdebstrap: Move preparations to hooks
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (4 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 5/8] mmdebstrap: Fix missing dpkg available Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 7/8] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
` (4 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
Also migrate from apt-key to gpg.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-mmdebstrap/isar-mmdebstrap.inc | 244 ++++++++----------
1 file changed, 114 insertions(+), 130 deletions(-)
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index e1c3832d..4128ad80 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -13,8 +13,6 @@ FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
ROOTFSDIR = "${WORKDIR}/rootfs"
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
# Fix for /var/lib/apt/available while maybe-jessie-or-older hook do not work
@@ -25,53 +23,29 @@ DPKG_HOOKS ?= "${@'--hook-dir='+d.getVar('MM_HOOK_JESSIE') \
MMHOOKS:ubuntu-focal ?= "${DPKG_HOOKS}"
MMHOOKS:debian-buster ?= "${DPKG_HOOKS}"
-def get_distro_primary_source_entry(d):
- for source in generate_distro_sources(d):
- if source[0] == "deb":
- return source[2:]
- bb.fatal('Invalid apt sources list')
-
-def get_distro_have_https_source(d):
- return any(source[2].startswith("https://") for source in generate_distro_sources(d))
-
-def get_distro_needs_https_support(d):
- if get_distro_have_https_source(d):
- return "https-support"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
-
-def get_distro_needs_gpg_support(d):
- if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
- d.getVar("THIRD_PARTY_APT_KEYS") or \
- d.getVar("BASE_REPO_KEY"):
- return "gnupg"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-
-APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
-do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
-do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
+do_generate_keyrings[dirs] = "${DEBDIR}"
do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
do_generate_keyrings[network] = "${TASK_USE_SUDO}"
do_generate_keyrings() {
+ export GNUPGHOME="$(mktemp -td gpghomeXXXXXXXXXX)"
if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
- chmod 777 "${APT_KEYS_DIR}"
for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ gpg --no-autostart --keyring "gnupg-ring:${DISTRO_BOOTSTRAP_KEYRING}" \
+ --no-default-keyring --import $keyfile
done
fi
if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
- sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ gpg --no-autostart --keyring "gnupg-ring:${DISTRO_BOOTSTRAP_KEYRING}" \
+ --no-default-keyring --import $keyfile
done
fi
+ if [ -r "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
+ chmod o+r "${DISTRO_BOOTSTRAP_KEYRING}"
+ fi
}
addtask generate_keyrings before do_build after do_unpack
@@ -80,10 +54,12 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
+DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -93,126 +69,134 @@ do_bootstrap() {
bootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
bootstrap_args="$bootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST
- deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${ROOTFSDIR}"
mkdir -p "${ROOTFSDIR}"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
+ bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
+
+ # no need to sync /var/cache/apt/archives if base-apt used
+ syncin='echo skip sync-in'
+ syncout='echo skip sync-out'
+ extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
+ extra_extract="$syncout"
+ # save mmdebstrap tempdir for cleanup
+ extra_essential="mkdir -p \$1/$base_apt_tmp && \
+ echo \$1 > ${WORKDIR}/mmtmpdir && \
+ mount -o bind,private '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
+ # replace base-apt mount in tmp with /base-apt mount
+ extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
+ \$1/etc/apt/sources.list.d/*.list && \
+ mkdir -p \$1/base-apt && \
+ mount -o bind,private '${REPO_BASE_DIR}' \$1/base-apt && \
+ chroot \$1 apt-get update -y \
+ -o APT::Update::Error-Mode=any && \
+ chroot \$1 apt-get install -y dpkg && \
+ umount \$1/base-apt && \
+ umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
+ umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
+ else
+ deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
+ install -v -m644 "${APTSRCS}" \
+ "${WORKDIR}/sources.list.d/bootstrap.list"
+
+ syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
+ "$1/var/cache/apt/archives/" || true'
+ syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "$1/var/cache/apt/archives/"*.deb \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
+ extra_setup="$syncin"
+ extra_extract="$syncout"
+ # prefetch apt debs because mmdebstrap will clean them on next stage
+ extra_essential='apt-get install apt -y -d \
+ -o Dir::State="$1/var/lib/apt" \
+ -o Dir::Etc="$1/etc/apt" \
+ -o Dir::Cache="$1/var/cache/apt" \
+ -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
+ extra_essential="$extra_essential && $syncout"
+ extra_customize="$syncout"
+ fi
+
+ if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
+ export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+ fi
+
arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
+ fi
+
+ # Cleanup mounts if fails
+ trap 'exit 1' INT HUP QUIT TERM ALRM USR1
+ trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
+ && rm "${WORKDIR}/mmtmpdir"; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
+ && sudo umount $tmpdir/$base_apt_tmp; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
+ && sudo umount $tmpdir/base-apt; \
+ [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
+ [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
+ && sudo umount $base_apt_tmp \
+ && rm -rf --one-file-system $base_apt_tmp' EXIT
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
$arch_param \
--mode=unshare \
${MMHOOKS} \
+ --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
+ --setup-hook="$extra_setup" \
+ --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
+ --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
+ --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
+ --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
+ --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
+ --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
+ --extract-hook="$extra_extract" \
+ --essential-hook="$extra_essential" \
+ --customize-hook="$extra_customize" \
+ --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
+ --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
+ --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
+ --skip=cleanup/apt \
+ --skip=download/empty \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
- "${@get_distro_source(d)}"
+ "$bootstrap_list"
sudo -E -s <<'EOSUDO'
set -e
tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
- # Install apt config
- mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
-
- mkdir -p ${ROOTFSDIR}/base-apt
- mount -o bind,private ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- else
- install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- fi
- install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
- rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
- find ${APT_KEYS_DIR}/ -type f | while read keyfile
- do
- MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
- echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
- export GNUPGHOME="${MY_GPGHOME}"
- APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
-
- kfn="$(basename $keyfile)"
- cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
- chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
- --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
- rm "${ROOTFSDIR}/tmp/$kfn"
-
- echo "Removing ${MY_GPGHOME}"
- rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
- done
-
- # Set locale
- install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
-
- sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-
- # update APT
- mount -o bind,private /dev ${ROOTFSDIR}/dev
- mount -o bind,private /dev/pts ${ROOTFSDIR}/dev/pts
- mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
- mount -t proc none ${ROOTFSDIR}/proc
- mount -o bind,private /sys ${ROOTFSDIR}/sys
- mount --make-rslave ${ROOTFSDIR}/sys
-
- export DEBIAN_FRONTEND=noninteractive
-
- if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
- fi
-
- if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
- fi
-
- chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
- -o APT::Update::Error-Mode=any
-
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
-
# setup chroot
- install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
"${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
- chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
- -o Debug::pkgProblemResolver=yes
-
- umount "${ROOTFSDIR}/dev/shm"
- umount "${ROOTFSDIR}/dev/pts"
- umount "${ROOTFSDIR}/dev"
- umount "${ROOTFSDIR}/proc"
- umount "${ROOTFSDIR}/sys"
- if mountpoint -q "${ROOTFSDIR}/base-apt"; then
- umount "${ROOTFSDIR}/base-apt"
- fi
-
# Finalize bootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
- deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
-
- # Cleanup apt cache
- sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
+ deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
+ fi
}
addtask bootstrap before do_build after do_generate_keyrings
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-7-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v11 7/8] isar-bootstrap: Use tar output instead of directory
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (5 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 6/8] mmdebstrap: Move preparations to hooks Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-06 8:21 ` [PATCH v11 8/8] user_manual.md: Update boostrap related documentation Anton Mikanovich
` (3 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Do not unpack tar with rootfs content inside bootstrap recipe.
This change can allow to remove sudo usage during the bootstrap later.
Using tar output requires moving chroot prepare step to the later
unpack step.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/rootfs.bbclass | 11 +++++---
.../isar-bootstrap/isar-bootstrap.inc | 25 ++++++++++---------
.../isar-mmdebstrap/isar-mmdebstrap.inc | 22 ++++------------
3 files changed, 25 insertions(+), 33 deletions(-)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index e339d24f..dee724a0 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -105,8 +105,8 @@ rootfs_do_qemu() {
fi
}
-BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
-BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
+BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
def rootfs_extra_import(d):
bb.utils._context["rootfs_progress"] = __import__("rootfs_progress")
@@ -116,7 +116,10 @@ ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}"
rootfs_prepare[weight] = "25"
rootfs_prepare(){
- sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+ sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # setup chroot
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
}
ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -275,7 +278,7 @@ cache_deb_src() {
# Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
# relies on that through out the build. Copy that state information instead of apt-get update
# which generates a new state from upstream.
- sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+ sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 0e5ab57d..f9dae26a 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -183,9 +183,8 @@ do_bootstrap() {
chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
- # setup chroot
+ # prepare setup chroot script
install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
@@ -199,8 +198,12 @@ do_bootstrap() {
mountpoint -q "${ROOTFSDIR}/base-apt" && \
umount "${ROOTFSDIR}/base-apt"
+ # Compress rootfs for compatibility
+ lopts="--one-file-system --exclude=var/cache/apt/archives"
+ tar --zstd -cf "${WORKDIR}/rootfs.tar.zst" $lopts -C "${ROOTFSDIR}" .
+
# Finalize debootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+ ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
EOSUDO
deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
@@ -216,19 +219,17 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- lopts="--one-file-system --exclude=var/cache/apt/archives"
- sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
- sudo chown $(id -u):$(id -g) bootstrap.tar
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
- # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
- # - after restoring from cache, there will be a tar which we unpack and then delete
- if [ -f bootstrap.tar ]; then
- sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
- sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
- rm bootstrap.tar
+ # we should restore symlinks after using tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 4128ad80..1043f2d1 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -11,7 +11,6 @@ inherit deb-dl-dir
FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-bootstrap/files"
-ROOTFSDIR = "${WORKDIR}/rootfs"
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
@@ -73,9 +72,6 @@ do_bootstrap() {
fi
E="${@ isar_export_proxies(d)}"
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
- mkdir -p "${ROOTFSDIR}"
-
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
@@ -182,17 +178,9 @@ do_bootstrap() {
"${WORKDIR}/rootfs.tar.zst" \
"$bootstrap_list"
- sudo -E -s <<'EOSUDO'
- set -e
-
- tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
-
- # setup chroot
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+ # Finalize bootstrap by setting the link in deploy
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
- # Finalize bootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
-EOSUDO
if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
@@ -206,7 +194,7 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
@@ -214,8 +202,8 @@ bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
# we should restore symlinks after using tar
if [ -f bootstrap.tar.zst ]; then
- mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
- sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
"${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-8-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v11 8/8] user_manual.md: Update boostrap related documentation
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (6 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 7/8] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
@ 2024-11-06 8:21 ` Anton Mikanovich
2024-11-07 6:26 ` [PATCH v11 0/8] Migrate to mmdebstrap 'MOESSBAUER, Felix' via isar-users
` (2 subsequent siblings)
10 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2024-11-06 8:21 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Update documentation in user manual and recipes API changelog.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
RECIPE-API-CHANGELOG.md | 13 +++++++++++++
doc/user_manual.md | 15 ++++++++++++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index 56ebfeb7..deb1c4e1 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -680,3 +680,16 @@ Bitbake exit status depends on ISAR_FAIL_ON_CLEANUP bitbake variable:
rootfs_do_umounts is not called from do_rootfs_finalize anymore.
Every individual task that does mounting must also do the umounting at its end.
+
+### Default boostrap recipe changed to mmdebstrap
+
+New virtual packages bootstrap-host and bootstrap-target are introduced.
+There are two providers of bootstrap-host/-target currently:
+ * isar-mmdebstrap: deafult one using mmdebstrap to prepare rootfs
+ * isar-bootstrap: previous bootstrap implementation left for compatibility
+
+So default bootstrap procedure is now performing with mmdebstrap.
+Previous implementation still can be selected by setting in local.conf:
+
+PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
+PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
diff --git a/doc/user_manual.md b/doc/user_manual.md
index 284847c8..06561247 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -66,7 +66,8 @@ The steps below describe how to build the images provided by default.
### Install Host Tools
-The supported host system is >= buster.
+The supported host system is >= buster for debootstrap and >= bullseye for
+default mmdebstrap provider.
Building `debian-trixie` requires host system >= bookworm.
@@ -75,7 +76,9 @@ Install the following packages:
apt install \
binfmt-support \
bzip2 \
- debootstrap \
+ mmdebstrap \
+ arch-test \
+ apt-utils \
dpkg-dev \
gettext-base \
git \
@@ -93,6 +96,12 @@ apt install \
zstd
```
+If using isar-bootstrap provider, debootstrap should be installed instead of
+mmdebstrap:
+```
+apt install debootstrap
+```
+
If your host is >= buster, also install the following package.
```
apt install python3-distutils
@@ -135,7 +144,7 @@ apt install qemu
### Setup Sudo
-Isar requires `sudo` rights without password to work with `chroot` and `debootstrap`. To add them, use the following steps:
+Isar requires `sudo` rights without password to work with `chroot`. To add them, use the following steps:
```
# visudo
```
--
2.34.1
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20241106082117.1089554-9-amikan%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 0/8] Migrate to mmdebstrap
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (7 preceding siblings ...)
2024-11-06 8:21 ` [PATCH v11 8/8] user_manual.md: Update boostrap related documentation Anton Mikanovich
@ 2024-11-07 6:26 ` 'MOESSBAUER, Felix' via isar-users
2024-11-14 8:50 ` Uladzimir Bely
2024-11-20 5:34 ` Uladzimir Bely
10 siblings, 0 replies; 15+ messages in thread
From: 'MOESSBAUER, Felix' via isar-users @ 2024-11-07 6:26 UTC (permalink / raw)
To: amikan, isar-users; +Cc: Heinisch, Alexander
On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
> Switch Isar from debootstrap to mmdebstrap.
>
> It makes isar-bootstrap target ~10..30% faster on both cross and
> native
> rootfs types. Some measurements on qemuarm64-bookworm (required debs
> are
> predownloaded and used during deb-dl-import):
>
> - 20s vs 28s for "isar-bootstrap-host"
> - 1m32s vs 1m48s for "isar-bootstrap-target
>
> Also mmdebstrap support using SOURCE_DATE_EPOCH for images
> reproducibly
> out of the box. But the main advantage is an ability to remove sudo
> for
> debootstrapping later on (this needs "uidmap" package installed).
Hi,
I'm really looking forward to switch to mmdebstrap. One of the biggest
advantage is that apt options can be passed - which are needed for the
snapshot mirrors. I'm also currently looking into optimizing the whole
chain to have reliable fetches from the snapshot mirrors. This includes
a fix of apt itself, which hopefully will get backported as well [1].
[1] https://salsa.debian.org/apt-team/apt/-/merge_requests/383
Best regards,
Felix
>
> This patchset was successfully tested with following downstreams:
> - cip-core
> - meta-iot2050
> - xenomai-images
>
> This change breaks Debian Buster support as host distro, but previous
> isar-bootstrap (with Buster support) still can be selected by setting
> in local.conf:
>
> PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
> PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
>
> Changes since v10:
> - Rebase on next.
>
> Changes since v9:
> - Rebase on next.
> - Separate common code into bbclass.
> - Skip deb cache on rootfs packing.
>
> Changes since v8:
> - Rebase on next.
> - Keep isar-bootstrap recipes.
> - Implement bootstrap provider selection.
>
> Changes since v7:
> - Rebase on next.
>
> Changes since v6:
> - Reorder patches.
>
> Changes since v5:
> - Improve keyring handling.
> - Fix mmdebstrap hooks.
>
> Changes since v4:
> - Rebase on next.
> - Migrate from apt-key to gpg.
> - Fix Ubuntu and RPi targets.
> - Cleanup debootstrap mentions.
> - Fix commit messages.
>
> Changes since v3:
> - Rebase on latest next.
> - Remove temporary patch 7 since kas:4.0 already includes mmdebstrap.
>
> Changes since v2:
> - Rebase on next.
> - Fix commit messages.
> - Update host requirements.
>
> Changes since v1:
> - Move mmdebstrap tmpdir to workdir.
> - Fix kas building.
> - Improve mounts cleanup.
>
> Anton Mikanovich (8):
> isar-bootstrap: Move common parts to bbclass
> meta: Add mmdebstrap recipe
> meta: Allow selecting bootstrap providers
> testsuite: Allow variable bootstrap providers
> mmdebstrap: Fix missing dpkg available
> mmdebstrap: Move preparations to hooks
> isar-bootstrap: Use tar output instead of directory
> user_manual.md: Update boostrap related documentation
>
> RECIPE-API-CHANGELOG.md | 13 +
> doc/user_manual.md | 15 +-
> meta-isar/conf/local.conf.sample | 4 +
> meta/classes/bootstrap.bbclass | 226 ++++++++++++++++
> meta/classes/rootfs.bbclass | 13 +-
> meta/conf/bitbake.conf | 3 +
> .../isar-bootstrap/isar-bootstrap-host.bb | 2 +
> .../isar-bootstrap/isar-bootstrap-target.bb | 2 +
> .../isar-bootstrap/isar-bootstrap.inc | 244 ++--------------
> --
> .../isar-mmdebstrap/isar-mmdebstrap-host.bb | 19 ++
> .../isar-mmdebstrap/isar-mmdebstrap-target.bb | 14 +
> .../isar-mmdebstrap/isar-mmdebstrap.inc | 216 ++++++++++++++++
> testsuite/cibase.py | 9 +-
> 13 files changed, 540 insertions(+), 240 deletions(-)
> create mode 100644 meta/classes/bootstrap.bbclass
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-host.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-target.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap.inc
>
> --
> 2.34.1
>
--
Siemens AG, Technology
Linux Expert Center
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/40d081a480766d25748d092b3aa47340e3ed342a.camel%40siemens.com.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 0/8] Migrate to mmdebstrap
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (8 preceding siblings ...)
2024-11-07 6:26 ` [PATCH v11 0/8] Migrate to mmdebstrap 'MOESSBAUER, Felix' via isar-users
@ 2024-11-14 8:50 ` Uladzimir Bely
2024-11-20 5:34 ` Uladzimir Bely
10 siblings, 0 replies; 15+ messages in thread
From: Uladzimir Bely @ 2024-11-14 8:50 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
> Switch Isar from debootstrap to mmdebstrap.
>
> It makes isar-bootstrap target ~10..30% faster on both cross and
> native
> rootfs types. Some measurements on qemuarm64-bookworm (required debs
> are
> predownloaded and used during deb-dl-import):
>
> - 20s vs 28s for "isar-bootstrap-host"
> - 1m32s vs 1m48s for "isar-bootstrap-target
>
> Also mmdebstrap support using SOURCE_DATE_EPOCH for images
> reproducibly
> out of the box. But the main advantage is an ability to remove sudo
> for
> debootstrapping later on (this needs "uidmap" package installed).
>
> This patchset was successfully tested with following downstreams:
> - cip-core
> - meta-iot2050
> - xenomai-images
>
> This change breaks Debian Buster support as host distro, but previous
> isar-bootstrap (with Buster support) still can be selected by setting
> in local.conf:
>
> PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
> PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
>
> Changes since v10:
> - Rebase on next.
>
> Changes since v9:
> - Rebase on next.
> - Separate common code into bbclass.
> - Skip deb cache on rootfs packing.
>
> Changes since v8:
> - Rebase on next.
> - Keep isar-bootstrap recipes.
> - Implement bootstrap provider selection.
>
> Changes since v7:
> - Rebase on next.
>
> Changes since v6:
> - Reorder patches.
>
> Changes since v5:
> - Improve keyring handling.
> - Fix mmdebstrap hooks.
>
> Changes since v4:
> - Rebase on next.
> - Migrate from apt-key to gpg.
> - Fix Ubuntu and RPi targets.
> - Cleanup debootstrap mentions.
> - Fix commit messages.
>
> Changes since v3:
> - Rebase on latest next.
> - Remove temporary patch 7 since kas:4.0 already includes mmdebstrap.
>
> Changes since v2:
> - Rebase on next.
> - Fix commit messages.
> - Update host requirements.
>
> Changes since v1:
> - Move mmdebstrap tmpdir to workdir.
> - Fix kas building.
> - Improve mounts cleanup.
>
> Anton Mikanovich (8):
> isar-bootstrap: Move common parts to bbclass
> meta: Add mmdebstrap recipe
> meta: Allow selecting bootstrap providers
> testsuite: Allow variable bootstrap providers
> mmdebstrap: Fix missing dpkg available
> mmdebstrap: Move preparations to hooks
> isar-bootstrap: Use tar output instead of directory
> user_manual.md: Update boostrap related documentation
>
> RECIPE-API-CHANGELOG.md | 13 +
> doc/user_manual.md | 15 +-
> meta-isar/conf/local.conf.sample | 4 +
> meta/classes/bootstrap.bbclass | 226 ++++++++++++++++
> meta/classes/rootfs.bbclass | 13 +-
> meta/conf/bitbake.conf | 3 +
> .../isar-bootstrap/isar-bootstrap-host.bb | 2 +
> .../isar-bootstrap/isar-bootstrap-target.bb | 2 +
> .../isar-bootstrap/isar-bootstrap.inc | 244 ++--------------
> --
> .../isar-mmdebstrap/isar-mmdebstrap-host.bb | 19 ++
> .../isar-mmdebstrap/isar-mmdebstrap-target.bb | 14 +
> .../isar-mmdebstrap/isar-mmdebstrap.inc | 216 ++++++++++++++++
> testsuite/cibase.py | 9 +-
> 13 files changed, 540 insertions(+), 240 deletions(-)
> create mode 100644 meta/classes/bootstrap.bbclass
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-host.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-target.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap.inc
>
> --
> 2.34.1
>
Hello all.
We would like to merge the patchset next week (Nov 19), if there a no
objections.
--
Best regards,
Uladzimir.
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/6f71354f13214c18b521046589c2b6e0e3276c79.camel%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v11 0/8] Migrate to mmdebstrap
2024-11-06 8:21 [PATCH v11 0/8] Migrate to mmdebstrap Anton Mikanovich
` (9 preceding siblings ...)
2024-11-14 8:50 ` Uladzimir Bely
@ 2024-11-20 5:34 ` Uladzimir Bely
10 siblings, 0 replies; 15+ messages in thread
From: Uladzimir Bely @ 2024-11-20 5:34 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
> Switch Isar from debootstrap to mmdebstrap.
>
> It makes isar-bootstrap target ~10..30% faster on both cross and
> native
> rootfs types. Some measurements on qemuarm64-bookworm (required debs
> are
> predownloaded and used during deb-dl-import):
>
> - 20s vs 28s for "isar-bootstrap-host"
> - 1m32s vs 1m48s for "isar-bootstrap-target
>
> Also mmdebstrap support using SOURCE_DATE_EPOCH for images
> reproducibly
> out of the box. But the main advantage is an ability to remove sudo
> for
> debootstrapping later on (this needs "uidmap" package installed).
>
> This patchset was successfully tested with following downstreams:
> - cip-core
> - meta-iot2050
> - xenomai-images
>
> This change breaks Debian Buster support as host distro, but previous
> isar-bootstrap (with Buster support) still can be selected by setting
> in local.conf:
>
> PREFERRED_PROVIDER_bootstrap-host ?= "isar-bootstrap-host"
> PREFERRED_PROVIDER_bootstrap-target ?= "isar-bootstrap-target"
>
> Changes since v10:
> - Rebase on next.
>
> Changes since v9:
> - Rebase on next.
> - Separate common code into bbclass.
> - Skip deb cache on rootfs packing.
>
> Changes since v8:
> - Rebase on next.
> - Keep isar-bootstrap recipes.
> - Implement bootstrap provider selection.
>
> Changes since v7:
> - Rebase on next.
>
> Changes since v6:
> - Reorder patches.
>
> Changes since v5:
> - Improve keyring handling.
> - Fix mmdebstrap hooks.
>
> Changes since v4:
> - Rebase on next.
> - Migrate from apt-key to gpg.
> - Fix Ubuntu and RPi targets.
> - Cleanup debootstrap mentions.
> - Fix commit messages.
>
> Changes since v3:
> - Rebase on latest next.
> - Remove temporary patch 7 since kas:4.0 already includes mmdebstrap.
>
> Changes since v2:
> - Rebase on next.
> - Fix commit messages.
> - Update host requirements.
>
> Changes since v1:
> - Move mmdebstrap tmpdir to workdir.
> - Fix kas building.
> - Improve mounts cleanup.
>
> Anton Mikanovich (8):
> isar-bootstrap: Move common parts to bbclass
> meta: Add mmdebstrap recipe
> meta: Allow selecting bootstrap providers
> testsuite: Allow variable bootstrap providers
> mmdebstrap: Fix missing dpkg available
> mmdebstrap: Move preparations to hooks
> isar-bootstrap: Use tar output instead of directory
> user_manual.md: Update boostrap related documentation
>
> RECIPE-API-CHANGELOG.md | 13 +
> doc/user_manual.md | 15 +-
> meta-isar/conf/local.conf.sample | 4 +
> meta/classes/bootstrap.bbclass | 226 ++++++++++++++++
> meta/classes/rootfs.bbclass | 13 +-
> meta/conf/bitbake.conf | 3 +
> .../isar-bootstrap/isar-bootstrap-host.bb | 2 +
> .../isar-bootstrap/isar-bootstrap-target.bb | 2 +
> .../isar-bootstrap/isar-bootstrap.inc | 244 ++--------------
> --
> .../isar-mmdebstrap/isar-mmdebstrap-host.bb | 19 ++
> .../isar-mmdebstrap/isar-mmdebstrap-target.bb | 14 +
> .../isar-mmdebstrap/isar-mmdebstrap.inc | 216 ++++++++++++++++
> testsuite/cibase.py | 9 +-
> 13 files changed, 540 insertions(+), 240 deletions(-)
> create mode 100644 meta/classes/bootstrap.bbclass
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-host.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-target.bb
> create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap.inc
>
> --
> 2.34.1
>
Applied to next.
--
Best regards,
Uladzimir.
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/b46f5ffa383e2149a4d5f4796af1ec4b1dd0e03f.camel%40ilbers.de.
^ permalink raw reply [flat|nested] 15+ messages in thread