Hi Alex, On Wed, 2018-03-14 at 17:04 +0300, Alexander Smirnov wrote: > > On 03/07/2018 07:44 PM, claudius.heine.ext@siemens.com wrote: > > From: Claudius Heine > > > > Since multistrap is deprecated for some years, it is required to > > change > > to debootstrap. > > > > This patch introduces the 'isar-bootstrap' recipe that implement > > the > > creation of a minimal base root file system using debootstrap. > > > > Signed-off-by: Claudius Heine > > --- > > meta-isar/conf/distro/debian-jessie.conf | 15 +- > > meta-isar/conf/distro/debian-jessie.list | 3 + > > meta-isar/conf/distro/debian-stretch.conf | 11 +- > > meta-isar/conf/distro/debian-stretch.list | 3 + > > meta-isar/conf/distro/debian-wheezy.conf | 11 +- > > meta-isar/conf/distro/debian-wheezy.list | 3 + > > meta-isar/conf/distro/raspbian-jessie.conf | 10 +- > > meta-isar/conf/distro/raspbian-jessie.list | 1 + > > .../isar-bootstrap/files/isar-apt.conf | 2 + > > meta/recipes-core/isar-bootstrap/isar-bootstrap.bb | 205 > > +++++++++++++++++++++ > > 10 files changed, 247 insertions(+), 17 deletions(-) > > create mode 100644 meta-isar/conf/distro/debian-jessie.list > > create mode 100644 meta-isar/conf/distro/debian-stretch.list > > create mode 100644 meta-isar/conf/distro/debian-wheezy.list > > create mode 100644 meta-isar/conf/distro/raspbian-jessie.list > > create mode 100644 meta/recipes-core/isar-bootstrap/files/isar- > > apt.conf > > create mode 100644 meta/recipes-core/isar-bootstrap/isar- > > bootstrap.bb > > > > diff --git a/meta-isar/conf/distro/debian-jessie.conf b/meta- > > isar/conf/distro/debian-jessie.conf > > index 3e62b14..df27fd7 100644 > > --- a/meta-isar/conf/distro/debian-jessie.conf > > +++ b/meta-isar/conf/distro/debian-jessie.conf > > @@ -1,16 +1,19 @@ > > -# Debian jessie multistrap configuration > > +# Debian jessie distribution configuration > > # > > # This software is a part of ISAR. > > # Copyright (C) 2015-2016 ilbers GmbH > > > > +DISTRO_APT_SOURCES += "conf/distro/debian-jessie.list" > > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > +DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 > > armmp \ > > + armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 > > octeon \ > > + orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 > > r5k-ip32 \ > > + s390x sb1-bcm91250a versatile" > > + > > +# No longer needed: > > DISTRO_SUITE ?= "jessie" > > DISTRO_COMPONENTS ?= "main contrib non-free" > > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-secur > > ity" > > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > > -DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 > > armmp \ > > - armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 > > octeon \ > > - orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 > > r5k-ip32 \ > > - s390x sb1-bcm91250a versatile" > > diff --git a/meta-isar/conf/distro/debian-jessie.list b/meta- > > isar/conf/distro/debian-jessie.list > > new file mode 100644 > > index 0000000..be46a57 > > --- /dev/null > > +++ b/meta-isar/conf/distro/debian-jessie.list > > @@ -0,0 +1,3 @@ > > +deb http://ftp.de.debian.org/debian class="Apple-tab-span" > > style="white-space:pre"> jessie main contrib non-free > > +deb http://ftp.de.debian.org/debian jessie-updates m > > ain contrib non-free > > +deb http://security.debian.org class="Apple-tab-span" > > style="white-space:pre"> jessie/updates main contrib non- > > free > > diff --git a/meta-isar/conf/distro/debian-stretch.conf b/meta- > > isar/conf/distro/debian-stretch.conf > > index 5a85d52..13c4f94 100644 > > --- a/meta-isar/conf/distro/debian-stretch.conf > > +++ b/meta-isar/conf/distro/debian-stretch.conf > > @@ -1,13 +1,16 @@ > > # This software is a part of ISAR. > > # Copyright (C) 2017 ilbers GmbH > > > > +DISTRO_APT_SOURCES += "conf/distro/debian-stretch.list" > > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > +DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 > > armmp \ > > + armmp-lpae kirkwood loongson-3 marvell octeon orion5x > > powerpc64le \ > > + rt-686-pae rt-amd64 s390x" > > + > > +# No longer needed: > > DISTRO_SUITE ?= "stretch" > > DISTRO_COMPONENTS ?= "main contrib non-free" > > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-secur > > ity" > > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > > -DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 > > armmp \ > > - armmp-lpae kirkwood loongson-3 marvell octeon orion5x > > powerpc64le \ > > - rt-686-pae rt-amd64 s390x" > > diff --git a/meta-isar/conf/distro/debian-stretch.list b/meta- > > isar/conf/distro/debian-stretch.list > > new file mode 100644 > > index 0000000..bfa5cf9 > > --- /dev/null > > +++ b/meta-isar/conf/distro/debian-stretch.list > > @@ -0,0 +1,3 @@ > > +deb http://ftp.de.debian.org/debian stretch main > > contrib non-free > > +deb http://ftp.de.debian.org/debian stretch-updates > > main contrib non-free > > +deb http://security.debian.org class="Apple-tab-span" > > style="white-space:pre"> stretch/updates main contrib non- > > free > > diff --git a/meta-isar/conf/distro/debian-wheezy.conf b/meta- > > isar/conf/distro/debian-wheezy.conf > > index f5ab6eb..2108f9b 100644 > > --- a/meta-isar/conf/distro/debian-wheezy.conf > > +++ b/meta-isar/conf/distro/debian-wheezy.conf > > @@ -1,14 +1,17 @@ > > -# Debian wheezy multistrap configuration > > +# Debian wheezy distribution configuration > > # > > # This software is a part of ISAR. > > # Copyright (C) 2015-2016 ilbers GmbH > > > > +DISTRO_APT_SOURCES += "conf/distro/debian-wheezy.list" > > +DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > +DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile > > vexpress \ > > + iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x" > > + > > +# No longer needed: > > DISTRO_SUITE ?= "wheezy" > > DISTRO_COMPONENTS ?= "main contrib non-free" > > DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian" > > DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-secur > > ity" > > -DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh" > > DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security" > > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > > -DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile > > vexpress \ > > - iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x" > > diff --git a/meta-isar/conf/distro/debian-wheezy.list b/meta- > > isar/conf/distro/debian-wheezy.list > > new file mode 100644 > > index 0000000..0bfa98e > > --- /dev/null > > +++ b/meta-isar/conf/distro/debian-wheezy.list > > @@ -0,0 +1,3 @@ > > +deb http://ftp.de.debian.org/debian wheezy main > > contrib non-free > > +deb http://ftp.de.debian.org/debian wheezy-updates m > > ain contrib non-free > > +deb http://security.debian.org class="Apple-tab-span" > > style="white-space:pre"> wheezy/updates main contrib non- > > free > > diff --git a/meta-isar/conf/distro/raspbian-jessie.conf b/meta- > > isar/conf/distro/raspbian-jessie.conf > > index c2a867d..57d0e7d 100644 > > --- a/meta-isar/conf/distro/raspbian-jessie.conf > > +++ b/meta-isar/conf/distro/raspbian-jessie.conf > > @@ -1,13 +1,17 @@ > > -# Raspbian stable multistrap configuration > > +# Raspbian stable distribution configuration > > # > > # This software is a part of ISAR. > > # Copyright (C) 2015-2016 ilbers GmbH > > > > +DISTRO_APT_SOURCES += "conf/distro/raspbian-jessie.list" > > +DISTRO_APT_KEYS += "https://archive.raspbian.org/raspbian.public.k > > ey;sha256sum=ca59cd4f2bcbc3a1d41ba6815a02a8dc5c175467a59bd87edeac45 > > 8f4a5345de" > > +DISTRO_CONFIG_SCRIPT?= "raspbian-configscript.sh" > > +DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv" > > + > > +# No longer needed: > > DISTRO_SUITE ?= "jessie" > > DISTRO_COMPONENTS ?= "main contrib non-free firmware" > > DISTRO_APT_SOURCE ?= "http://archive.raspbian.org/raspbian" > > DISTRO_APT_SOURCE_SEC ?= "" > > -DISTRO_CONFIG_SCRIPT ?= "raspbian-configscript.sh" > > DISTRO_MULTICONF_BOOTSTRAP ?= "base" > > DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}" > > -DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv" > > diff --git a/meta-isar/conf/distro/raspbian-jessie.list b/meta- > > isar/conf/distro/raspbian-jessie.list > > new file mode 100644 > > index 0000000..792f9c6 > > --- /dev/null > > +++ b/meta-isar/conf/distro/raspbian-jessie.list > > @@ -0,0 +1 @@ > > +deb http://archive.raspbian.org/raspbian class="Apple-tab- > > span" style="white-space:pre"> jessie main contrib non-free > > firmware > > diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > > b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > > new file mode 100644 > > index 0000000..a62feb1 > > --- /dev/null > > +++ b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf > > @@ -0,0 +1,2 @@ > > +APT::Install-Recommends "0"; > > +APT::Install-Suggests "0"; > > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > > new file mode 100644 > > index 0000000..2ddefa4 > > --- /dev/null > > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb > > @@ -0,0 +1,205 @@ > > +# Minimal debian root file system > > +# > > +# This software is a part of ISAR. > > +# Copyright (c) Siemens AG, 2018 > > +# > > +# SPDX-License-Identifier: MIT > > + > > +Description = "Minimal debian root file system" > > + > > +LICENSE = "gpl-2.0" > > +LIC_FILES_CHKSUM = > > "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa9544 > > 99f7abaabaa882bbe" > > +FILESPATH_prepend := "${THISDIR}/files:" > > +SRC_URI = "file://isar-apt.conf" > > +PV = "1.0" > > + > > +WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}" > > +DEBOOTSTRAP ?= "" > > +ROOTFSDIR = "${WORKDIR}/rootfs" > > +APTPREFS = "${WORKDIR}/apt-preferences" > > +APTSRCS = "${WORKDIR}/apt-sources" > > +APTKEYFILES = "" > > +APTKEYRING = "${WORKDIR}/apt-keyring.gpg" > > +DEBOOTSTRAP_KEYRING = "" > > + > > +python () { > > + from urllib.parse import urlparse > > + > > + debootstrap = d.getVar("DEBOOTSTRAP", True) > > + if not debootstrap: > > + target = d.getVar("DISTRO_ARCH", True) > > + machine = os.uname()[4] > > + m = { > > + "x86_64": ["i386", "amd64"], > > + "x86": ["i386"], > > + } > > + if machine not in m or target not in m[machine]: > > + debootstrap = "qemu-debootstrap" > > + else: > > + debootstrap = "debootstrap" > > + d.setVar("DEBOOTSTRAP", debootstrap) > > + > > + distro_apt_keys = d.getVar("DISTRO_APT_KEYS", False) > > + if distro_apt_keys: > > + d.setVar("DEBOOTSTRAP_KEYRING", "--keyring ${APTKEYRING}") > > + for key in distro_apt_keys.split(): > > + url = urlparse(key) > > + filename = os.path.basename(url.path) > > + d.appendVar("SRC_URI", " %s" % key) > > + d.appendVar("APTKEYFILES", " %s" % filename) > > +} > > + > > +def aggregate_files(d, file_list, file_out): > > + import shutil > > + > > + with open(file_out, "wb") as out_fd: > > + for entry in file_list: > > + entry_real = bb.parse.resolve_file(entry, d) > > + with open(entry_real, "rb") as in_fd: > > + shutil.copyfileobj(in_fd, out_fd, 1024*1024*10) > > + out_fd.write("\n".encode()) > > + > > +def parse_aptsources_list_line(source_list_line): > > + import re > > + > > + s = source_list_line.strip() > > + > > + if s.startswith("#"): > > + return None > > + > > + type, s = re.split("\s+", s, maxsplit=1) > > + if type not in ["deb", "deb-src"]: > > + return None > > + > > + options = "" > > + options_match = > > re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s) > > + if options_match: > > + options = options_match.group(0).strip() > > + s = s[options_match.end():] > > + > > + source, s = re.split("\s+", s, maxsplit=1) > > + > > + suite, s = re.split("\s+", s, maxsplit=1) > > + > > + components = " ".join(s.split()) > > + > > + return type, options, source, suite, components > > + > > +def get_distro_primary_source_entry(d): > > + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or > > "").split() > > + for entry in apt_sources_list: > > + entry_real = bb.parse.resolve_file(entry, d) > > + with open(entry_real, "r") as in_fd: > > + for line in in_fd: > > + parsed = parse_aptsources_list_line(line) > > + if parsed: > > + type, _, source, suite, components = parsed > > + if type == "deb": > > + return source, suite, components > > + return "", "", "" > > + > > +def get_distro_source(d): > > + return get_distro_primary_source_entry(d)[0] > > + > > +def get_distro_suite(d): > > + return get_distro_primary_source_entry(d)[1] > > + > > +def get_distro_components_argument(d): > > + components = get_distro_primary_source_entry(d)[2] > > + if components and components.strip(): > > + return "--components=%s" % ",".join(components.split()) > > + else: > > + return "" > > + > > +do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > > +do_generate_keyring[dirs] = "${WORKDIR}" > > +do_generate_keyring[vardeps] += "DISTRO_APT_KEYS" > > +do_generate_keyring() { > > + if [ -n "${@d.getVar("APTKEYFILES", True) or ""}" ]; then > > + for keyfile in ${@d.getVar("APTKEYFILES", True)}; do > > + gpg --no-default-keyring --keyring "${APTKEYRING}" -- > > import "$keyfile" > > This code touches my private "~/.gnupg" folder's content, what is > not > desirable behavior. Isar should touch files *only* in "build/tmp" > folder. Are you sure? $ find .gnupg -type f | sort | xargs b2sum > gpgsums.a $ gpg --no-default-keyring --keyring ./test1.db --import raspbian.public.key gpg: enabled debug flags: memstat gpg: keybox './test1.db' created gpg: key 9165938D90FDDD2E: public key "Mike Thompson (Raspberry Pi Debian armhf ARMv6+VFP) " imported gpg: Total number processed: 1 gpg: imported: 1 gpg: keydb: handles=3 locks=2 parse=2 get=2 gpg: build=1 update=0 insert=1 delete=0 gpg: reset=1 found=2 not=2 cache=0 not=0 gpg: kid_not_found_cache: count=0 peak=0 flushes=0 gpg: sig_cache: total=6 cached=4 good=4 bad=0 gpg: random usage: poolsize=600 mixed=0 polls=0/0 added=0/0 outmix=0 getlvl1=0/0 getlvl2=0/0 gpg: rndjent stat: collector=0x0000000000000000 calls=0 bytes=0 gpg: secmem usage: 0/32768 bytes in 0 blocks $ find .gnupg -type f | sort | xargs b2sum > gpgsums.b $ diff -u gpgsums.a gpgsums.b [no output] I could add the '--homedir' parameter, but normally it shouldn't use it because I manually select a different keyring. Claudius > > gpg already supports changing of home directory using --homedir > option. > So I think we could create dedicated folder "${TMPDIR}/gpg" and > store > keys here. > > At the moment I'm testing if debootstrap supports custom gpg > storage. > Will come back with the results. > > Alex > > > + done > > + fi > > +} > > +addtask generate_keyring before do_build after do_unpack > > + > > +do_apt_config_prepare[stamp-extra-info] = "${DISTRO}- > > ${DISTRO_ARCH}" > > +do_apt_config_prepare[vardeps] += "\ > > + APTPREFS \ > > + DISTRO_APT_PREFERENCES \ > > + DEBDISTRONAME \ > > + APTSRCS \ > > + DISTRO_APT_SOURCES \ > > + " > > +python do_apt_config_prepare() { > > + apt_preferences_out = d.getVar("APTPREFS", True) > > + apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", > > True) or "" > > + ).split() > > + aggregate_files(d, apt_preferences_list, apt_preferences_out) > > + > > + apt_sources_out = d.getVar("APTSRCS", True) > > + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or > > "").split() > > + > > + aggregate_files(d, apt_sources_list, apt_sources_out) > > +} > > +addtask apt_config_prepare before do_build after > > do_generate_keyring > > + > > +do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > > +do_bootstrap[vardeps] += "DISTRO_APT_SOURCES" > > +do_bootstrap() { > > + if [ -e "${ROOTFSDIR}" ]; then > > + sudo umount -l "${ROOTFSDIR}/dev" || true > > + sudo umount -l "${ROOTFSDIR}/proc" || true > > + sudo rm -rf "${ROOTFSDIR}" > > + fi > > + E="${@bb.utils.export_proxies(d)}" > > + sudo -E "${DEBOOTSTRAP}" --verbose \ > > + --variant minbase \ > > + --arch "${DISTRO_ARCH}" \ > > + ${@get_distro_components_argument(d)} > > \ > > + ${DEBOOTSTRAP_KEYRING} \ > > + "${@get_distro_suite(d)}" \ > > + "${ROOTFSDIR}" \ > > + "${@get_distro_source(d)}" > > +} > > +addtask bootstrap before do_build after do_apt_config_prepare > > + > > +do_apt_config_install[stamp-extra-info] = "${DISTRO}- > > ${DISTRO_ARCH}" > > +do_apt_config_install() { > > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d" > > + sudo install -v -m644 "${APTPREFS}" \ > > + "${ROOTFSDIR}/etc/apt/preferences.d/boot > > strap" > > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d" > > + sudo install -v -m644 "${APTSRCS}" \ > > + "${ROOTFSDIR}/etc/apt/sources.list.d/boo > > tstrap.list" > > + sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list" > > + sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d" > > + sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \ > > + "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar. > > conf" > > +} > > +addtask apt_config_install before do_build after do_bootstrap > > + > > +do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > > +do_apt_update() { > > + sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs > > ${ROOTFSDIR}/dev > > + sudo mount -t proc none ${ROOTFSDIR}/proc > > + > > + E="${@bb.utils.export_proxies(d)}" > > + export DEBIAN_FRONTEND=noninteractive > > + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y > > + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y > > \ > > + -o > > Debug::pkgProblemResolver=yes > > +} > > +addtask apt_update before do_build after do_apt_config_install > > + > > +do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" > > +do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}" > > +do_deploy() { > > + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}- > > ${DISTRO_ARCH}" > > +} > > +addtask deploy before do_build after do_apt_update > > + > > +CLEANFUNCS = "clean_deploy" > > +clean_deploy() { > > + rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}" > > +} > > > > -- DENX Software Engineering GmbH, Managing Director: Wolfgang Denk HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de PGP key: 6FF2 E59F 00C6 BC28 31D8 64C1 1173 CB19 9808 B153 Keyserver: hkp://pool.sks-keyservers.net