Hi, just reviewed it with a fresh mind. There are still some places where stuff can be optimized. But IMO that could also be done later. I might not have time to do it this month myself. Here are some of my points: On Wed, 2018-03-07 at 17:44 +0100, claudius.heine.ext@siemens.com wrote: > From: Claudius Heine > > Since multistrap is deprecated for some years, it is required to > change > to debootstrap. > > This patch introduces the 'isar-bootstrap' recipe that implement the > creation of a minimal base root file system using debootstrap. > > Signed-off-by: Claudius Heine > --- > meta-isar/conf/distro/debian-jessie.conf | 15 +- > meta-isar/conf/distro/debian-jessie.list | 3 + > meta-isar/conf/distro/debian-stretch.conf | 11 +- > meta-isar/conf/distro/debian-stretch.list | 3 + > meta-isar/conf/distro/debian-wheezy.conf | 11 +- > meta-isar/conf/distro/debian-wheezy.list | 3 + > meta-isar/conf/distro/raspbian-jessie.conf | 10 +- > meta-isar/conf/distro/raspbian-jessie.list | 1 + > .../isar-bootstrap/files/isar-apt.conf | 2 + > meta/recipes-core/isar-bootstrap/isar-bootstrap.bb | 205 > +++++++++++++++++++++ > 10 files changed, 247 insertions(+), 17 deletions(-) > create mode 100644 meta-isar/conf/distro/debian-jessie.list > create mode 100644 meta-isar/conf/distro/debian-stretch.list > create mode 100644 meta-isar/conf/distro/debian-wheezy.list > create mode 100644 meta-isar/conf/distro/raspbian-jessie.list > create mode 100644 meta/recipes-core/isar-bootstrap/files/isar- > apt.conf > create mode 100644 meta/recipes-core/isar-bootstrap/isar- > bootstrap.bb > > diff --git a/meta-isar/conf/distro/debian-jessie.conf b/meta- > isar/conf/distro/debian-jessie.conf > index 3e62b14..df27fd7 100644 > --- a/meta-isar/conf/distro/debian-jessie.conf > +++ b/meta-isar/conf/distro/debian-jessie.conf > @@ -1,16 +1,19 @@ > -# Debian jessie multistrap configuration > +# Debian jessie distribution configuration > # > # This software is a part of ISAR. > # Copyright (C) 2015-2016 ilbers GmbH > > +DISTRO_APT_SOURCES += "conf/distro/debian-jessie.list" > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > +DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 > armmp \ > + armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 > octeon \ > + orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 r5k- > ip32 \ > + s390x sb1-bcm91250a versatile" > + > +# No longer needed: > DISTRO_SUITE ?= "jessie" > DISTRO_COMPONENTS ?= "main contrib non-free" > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security > " > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > -DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 > armmp \ > - armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 > octeon \ > - orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 r5k- > ip32 \ > - s390x sb1-bcm91250a versatile" > diff --git a/meta-isar/conf/distro/debian-jessie.list b/meta- > isar/conf/distro/debian-jessie.list > new file mode 100644 > index 0000000..be46a57 > --- /dev/null > +++ b/meta-isar/conf/distro/debian-jessie.list > @@ -0,0 +1,3 @@ > +deb http://ftp.de.debian.org/debian class="Apple-tab-span" > style="white-space:pre"> jessie main contrib non-free > +deb http://ftp.de.debian.org/debian jessie-updates mai > n contrib non-free > +deb http://security.debian.org class="Apple-tab-span" > style="white-space:pre"> jessie/updates main contrib non-free > diff --git a/meta-isar/conf/distro/debian-stretch.conf b/meta- > isar/conf/distro/debian-stretch.conf > index 5a85d52..13c4f94 100644 > --- a/meta-isar/conf/distro/debian-stretch.conf > +++ b/meta-isar/conf/distro/debian-stretch.conf > @@ -1,13 +1,16 @@ > # This software is a part of ISAR. > # Copyright (C) 2017 ilbers GmbH > > +DISTRO_APT_SOURCES += "conf/distro/debian-stretch.list" > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > +DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 > armmp \ > + armmp-lpae kirkwood loongson-3 marvell octeon orion5x > powerpc64le \ > + rt-686-pae rt-amd64 s390x" > + > +# No longer needed: > DISTRO_SUITE ?= "stretch" > DISTRO_COMPONENTS ?= "main contrib non-free" > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security > " > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > -DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 > armmp \ > - armmp-lpae kirkwood loongson-3 marvell octeon orion5x > powerpc64le \ > - rt-686-pae rt-amd64 s390x" > diff --git a/meta-isar/conf/distro/debian-stretch.list b/meta- > isar/conf/distro/debian-stretch.list > new file mode 100644 > index 0000000..bfa5cf9 > --- /dev/null > +++ b/meta-isar/conf/distro/debian-stretch.list > @@ -0,0 +1,3 @@ > +deb http://ftp.de.debian.org/debian stretch main > contrib non-free > +deb http://ftp.de.debian.org/debian stretch-updates ma > in contrib non-free > +deb http://security.debian.org class="Apple-tab-span" > style="white-space:pre"> stretch/updates main contrib non-free > diff --git a/meta-isar/conf/distro/debian-wheezy.conf b/meta- > isar/conf/distro/debian-wheezy.conf > index f5ab6eb..2108f9b 100644 > --- a/meta-isar/conf/distro/debian-wheezy.conf > +++ b/meta-isar/conf/distro/debian-wheezy.conf > @@ -1,14 +1,17 @@ > -# Debian wheezy multistrap configuration > +# Debian wheezy distribution configuration > # > # This software is a part of ISAR. > # Copyright (C) 2015-2016 ilbers GmbH > > +DISTRO_APT_SOURCES += "conf/distro/debian-wheezy.list" > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > +DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile > vexpress \ > + iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x" > + > +# No longer needed: > DISTRO_SUITE ?= "wheezy" > DISTRO_COMPONENTS ?= "main contrib non-free" > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security > " > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > -DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile > vexpress \ > - iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x" > diff --git a/meta-isar/conf/distro/debian-wheezy.list b/meta- > isar/conf/distro/debian-wheezy.list > new file mode 100644 > index 0000000..0bfa98e > --- /dev/null > +++ b/meta-isar/conf/distro/debian-wheezy.list > @@ -0,0 +1,3 @@ > +deb http://ftp.de.debian.org/debian wheezy main > contrib non-free > +deb http://ftp.de.debian.org/debian wheezy-updates mai > n contrib non-free > +deb http://security.debian.org class="Apple-tab-span" > style="white-space:pre"> wheezy/updates main contrib non-free > diff --git a/meta-isar/conf/distro/raspbian-jessie.conf b/meta- > isar/conf/distro/raspbian-jessie.conf > index c2a867d..57d0e7d 100644 > --- a/meta-isar/conf/distro/raspbian-jessie.conf > +++ b/meta-isar/conf/distro/raspbian-jessie.conf > @@ -1,13 +1,17 @@ > -# Raspbian stable multistrap configuration > +# Raspbian stable distribution configuration > # > # This software is a part of ISAR. > # Copyright (C) 2015-2016 ilbers GmbH > > +DISTRO_APT_SOURCES += "conf/distro/raspbian-jessie.list" > +DISTRO_APT_KEYS += "https://archive.raspbian.org/raspbian.public.key > ;sha256sum=ca59cd4f2bcbc3a1d41ba6815a02a8dc5c175467a59bd87edeac458f4a > 5345de" > +DISTRO_CONFIG_SCRIPT?= "raspbian-configscript.sh" > +DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv" > + > +# No longer needed: > DISTRO_SUITE ?= "jessie" > DISTRO_COMPONENTS ?= "main contrib non-free firmware" > DISTRO_APT_SOURCE ?= "http://archive.raspbian.org/raspbian" > DISTRO_APT_SOURCE_SEC ?= "" > -DISTRO_CONFIG_SCRIPT ?= "raspbian-configscript.sh" > DISTRO_MULTICONF_BOOTSTRAP ?= "base" > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > -DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv" > diff --git a/meta-isar/conf/distro/raspbian-jessie.list b/meta- > isar/conf/distro/raspbian-jessie.list > new file mode 100644 > index 0000000..792f9c6 > --- /dev/null > +++ b/meta-isar/conf/distro/raspbian-jessie.list > @@ -0,0 +1 @@ > +deb http://archive.raspbian.org/raspbian class="Apple-tab- > span" style="white-space:pre"> jessie main contrib non-free > firmware > diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > new file mode 100644 > index 0000000..a62feb1 > --- /dev/null > +++ b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > @@ -0,0 +1,2 @@ > +APT::Install-Recommends "0"; > +APT::Install-Suggests "0"; > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > new file mode 100644 > index 0000000..2ddefa4 > --- /dev/null > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > @@ -0,0 +1,205 @@ > +# Minimal debian root file system > +# > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2018 > +# > +# SPDX-License-Identifier: MIT > + > +Description = "Minimal debian root file system" > + > +LICENSE = "gpl-2.0" > +LIC_FILES_CHKSUM = > "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499 > f7abaabaa882bbe" > +FILESPATH_prepend := "${THISDIR}/files:" > +SRC_URI = "file://isar-apt.conf" > +PV = "1.0" > + > +WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}" > +DEBOOTSTRAP ?= "" > +ROOTFSDIR = "${WORKDIR}/rootfs" > +APTPREFS = "${WORKDIR}/apt-preferences" > +APTSRCS = "${WORKDIR}/apt-sources" > +APTKEYFILES = "" > +APTKEYRING = "${WORKDIR}/apt-keyring.gpg" > +DEBOOTSTRAP_KEYRING = "" > + > +python () { > + from urllib.parse import urlparse > + > + debootstrap = d.getVar("DEBOOTSTRAP", True) > + if not debootstrap: > + target = d.getVar("DISTRO_ARCH", True) > + machine = os.uname()[4] > + m = { > + "x86_64": ["i386", "amd64"], > + "x86": ["i386"], > + } > + if machine not in m or target not in m[machine]: > + debootstrap = "qemu-debootstrap" > + else: > + debootstrap = "debootstrap" > + d.setVar("DEBOOTSTRAP", debootstrap) > + > + distro_apt_keys = d.getVar("DISTRO_APT_KEYS", False) > + if distro_apt_keys: > + d.setVar("DEBOOTSTRAP_KEYRING", "--keyring ${APTKEYRING}") > + for key in distro_apt_keys.split(): > + url = urlparse(key) > + filename = os.path.basename(url.path) > + d.appendVar("SRC_URI", " %s" % key) > + d.appendVar("APTKEYFILES", " %s" % filename) > +} > + > +def aggregate_files(d, file_list, file_out): > + import shutil > + > + with open(file_out, "wb") as out_fd: > + for entry in file_list: > + entry_real = bb.parse.resolve_file(entry, d) > + with open(entry_real, "rb") as in_fd: > + shutil.copyfileobj(in_fd, out_fd, 1024*1024*10) > + out_fd.write("\n".encode()) > + > +def parse_aptsources_list_line(source_list_line): > + import re > + > + s = source_list_line.strip() > + > + if s.startswith("#"): if not s or s.startswith("#"): That way empty lines will be ignored as well. > + return None > + > + type, s = re.split("\s+", s, maxsplit=1) s.split(maxsplit=1) No need for regex. > + if type not in ["deb", "deb-src"]: > + return None > + > + options = "" > + options_match = > re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s) Overly complicated regex. Something like this might be good enough: "\[[^\]]*]\s+" I written the complicated one first, because I wanted to extract all the individual options, but that is not really necessary now and maybe ever. > + if options_match: > + options = options_match.group(0).strip() > + s = s[options_match.end():] > + > + source, s = re.split("\s+", s, maxsplit=1) > + > + suite, s = re.split("\s+", s, maxsplit=1) Here re.split can be replaced with just split as well. > + > + components = " ".join(s.split()) > + > + return type, options, source, suite, components This whole function could also be simplified to just one regex. I originally tried that, but I wanted to extract the information completely as seen in the one real regex left. A regex like this could work: r"(?Pdeb|deb- src)\s+(?P\[[^\]]*]\s+)?(?P\S+)\s+(?P\S+)\s+(?P .*)" > + > +def get_distro_primary_source_entry(d): > + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or > "").split() > + for entry in apt_sources_list: > + entry_real = bb.parse.resolve_file(entry, d) > + with open(entry_real, "r") as in_fd: > + for line in in_fd: > + parsed = parse_aptsources_list_line(line) > + if parsed: > + type, _, source, suite, components = parsed > + if type == "deb": > + return source, suite, components > + return "", "", "" Maybe extract an iterator out of this function, and use this when generating the bootstrap.list file in do_apt_config_prepare as well. This way parse_aptsources_list_line or the interator itself could later be used to implement PREMIRRORS, similar to the PREMIRRORS in OE. Cheers, Claudius > + > +def get_distro_source(d): > + return get_distro_primary_source_entry(d)[0] > + > +def get_distro_suite(d): > + return get_distro_primary_source_entry(d)[1] > + > +def get_distro_components_argument(d): > + components = get_distro_primary_source_entry(d)[2] > + if components and components.strip(): > + return "--components=%s" % ",".join(components.split()) > + else: > + return "" > + > +do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_generate_keyring[dirs] = "${WORKDIR}" > +do_generate_keyring[vardeps] += "DISTRO_APT_KEYS" > +do_generate_keyring() { > + if [ -n "${@d.getVar("APTKEYFILES", True) or ""}" ]; then > + for keyfile in ${@d.getVar("APTKEYFILES", True)}; do > + gpg --no-default-keyring --keyring "${APTKEYRING}" -- > import "$keyfile" > + done > + fi > +} > +addtask generate_keyring before do_build after do_unpack > + > +do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_apt_config_prepare[vardeps] += "\ > + APTPREFS \ > + DISTRO_APT_PREFERENCES \ > + DEBDISTRONAME \ > + APTSRCS \ > + DISTRO_APT_SOURCES \ > + " > +python do_apt_config_prepare() { > + apt_preferences_out = d.getVar("APTPREFS", True) > + apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) > or "" > + ).split() > + aggregate_files(d, apt_preferences_list, apt_preferences_out) > + > + apt_sources_out = d.getVar("APTSRCS", True) > + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or > "").split() > + > + aggregate_files(d, apt_sources_list, apt_sources_out) > +} > +addtask apt_config_prepare before do_build after do_generate_keyring > + > +do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_bootstrap[vardeps] += "DISTRO_APT_SOURCES" > +do_bootstrap() { > + if [ -e "${ROOTFSDIR}" ]; then > + sudo umount -l "${ROOTFSDIR}/dev" || true > + sudo umount -l "${ROOTFSDIR}/proc" || true > + sudo rm -rf "${ROOTFSDIR}" > + fi > + E="${@bb.utils.export_proxies(d)}" > + sudo -E "${DEBOOTSTRAP}" --verbose \ > + --variant minbase \ > + --arch "${DISTRO_ARCH}" \ > + ${@get_distro_components_argument(d)} \ > + ${DEBOOTSTRAP_KEYRING} \ > + "${@get_distro_suite(d)}" \ > + "${ROOTFSDIR}" \ > + "${@get_distro_source(d)}" > +} > +addtask bootstrap before do_build after do_apt_config_prepare > + > +do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_apt_config_install() { > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d" > + sudo install -v -m644 "${APTPREFS}" \ > + "${ROOTFSDIR}/etc/apt/preferences.d/bootst > rap" > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d" > + sudo install -v -m644 "${APTSRCS}" \ > + "${ROOTFSDIR}/etc/apt/sources.list.d/boots > trap.list" > + sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list" > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d" > + sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \ > + "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.co > nf" > +} > +addtask apt_config_install before do_build after do_bootstrap > + > +do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_apt_update() { > + sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs > ${ROOTFSDIR}/dev > + sudo mount -t proc none ${ROOTFSDIR}/proc > + > + E="${@bb.utils.export_proxies(d)}" > + export DEBIAN_FRONTEND=noninteractive > + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y > + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \ > + -o > Debug::pkgProblemResolver=yes > +} > +addtask apt_update before do_build after do_apt_config_install > + > +do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > +do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}" > +do_deploy() { > + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}- > ${DISTRO_ARCH}" > +} > +addtask deploy before do_build after do_apt_update > + > +CLEANFUNCS = "clean_deploy" > +clean_deploy() { > + rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}" > +} -- DENX Software Engineering GmbH, Managing Director: Wolfgang Denk HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de PGP key: 6FF2 E59F 00C6 BC28 31D8 64C1 1173 CB19 9808 B153 Keyserver: hkp://pool.sks-keyservers.net