[PATCH v5 1/5] implement isar-bootstrap using debootstrap

[claudius.heine.ext@siemens.com]

From: Claudius Heine <ch@denx.de>

Since multistrap is deprecated for some years, it is required to change
to debootstrap.

This patch introduces the 'isar-bootstrap' recipe that implement the
creation of a minimal base root file system using debootstrap.

Signed-off-by: Claudius Heine <ch@denx.de>
---
 meta-isar/conf/distro/debian-jessie.conf           |  15 +-
 meta-isar/conf/distro/debian-jessie.list           |   3 +
 meta-isar/conf/distro/debian-stretch.conf          |  11 +-
 meta-isar/conf/distro/debian-stretch.list          |   3 +
 meta-isar/conf/distro/debian-wheezy.conf           |  11 +-
 meta-isar/conf/distro/debian-wheezy.list           |   3 +
 meta-isar/conf/distro/raspbian-jessie.conf         |  10 +-
 meta-isar/conf/distro/raspbian-jessie.list         |   1 +
 .../isar-bootstrap/files/isar-apt.conf             |   2 +
 meta/recipes-core/isar-bootstrap/isar-bootstrap.bb | 206 +++++++++++++++++++++
 10 files changed, 248 insertions(+), 17 deletions(-)
 create mode 100644 meta-isar/conf/distro/debian-jessie.list
 create mode 100644 meta-isar/conf/distro/debian-stretch.list
 create mode 100644 meta-isar/conf/distro/debian-wheezy.list
 create mode 100644 meta-isar/conf/distro/raspbian-jessie.list
 create mode 100644 meta/recipes-core/isar-bootstrap/files/isar-apt.conf
 create mode 100644 meta/recipes-core/isar-bootstrap/isar-bootstrap.bb

diff --git a/meta-isar/conf/distro/debian-jessie.conf b/meta-isar/conf/distro/debian-jessie.conf
index 3e62b14..df27fd7 100644
--- a/meta-isar/conf/distro/debian-jessie.conf
+++ b/meta-isar/conf/distro/debian-jessie.conf
@@ -1,16 +1,19 @@
-# Debian jessie multistrap configuration
+# Debian jessie distribution configuration
 #
 # This software is a part of ISAR.
 # Copyright (C) 2015-2016 ilbers GmbH
 
+DISTRO_APT_SOURCES += "conf/distro/debian-jessie.list"
+DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
+DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 armmp \
+    armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 octeon \
+    orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 r5k-ip32 \
+    s390x sb1-bcm91250a versatile"
+
+# No longer needed:
 DISTRO_SUITE ?= "jessie"
 DISTRO_COMPONENTS ?= "main contrib non-free"
 DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
 DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
-DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
 DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
 DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
-DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 armmp \
-    armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 octeon \
-    orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 r5k-ip32 \
-    s390x sb1-bcm91250a versatile"
diff --git a/meta-isar/conf/distro/debian-jessie.list b/meta-isar/conf/distro/debian-jessie.list
new file mode 100644
index 0000000..be46a57
--- /dev/null
+++ b/meta-isar/conf/distro/debian-jessie.list
@@ -0,0 +1,3 @@
+deb	http://ftp.de.debian.org/debian	jessie	main contrib non-free
+deb	http://ftp.de.debian.org/debian jessie-updates	main contrib non-free
+deb	http://security.debian.org	jessie/updates	main contrib non-free
diff --git a/meta-isar/conf/distro/debian-stretch.conf b/meta-isar/conf/distro/debian-stretch.conf
index 5a85d52..13c4f94 100644
--- a/meta-isar/conf/distro/debian-stretch.conf
+++ b/meta-isar/conf/distro/debian-stretch.conf
@@ -1,13 +1,16 @@
 # This software is a part of ISAR.
 # Copyright (C) 2017 ilbers GmbH
 
+DISTRO_APT_SOURCES += "conf/distro/debian-stretch.list"
+DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
+DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 armmp \
+    armmp-lpae kirkwood loongson-3 marvell octeon orion5x powerpc64le \
+    rt-686-pae rt-amd64 s390x"
+
+# No longer needed:
 DISTRO_SUITE ?= "stretch"
 DISTRO_COMPONENTS ?= "main contrib non-free"
 DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
 DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
-DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
 DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
 DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
-DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 armmp \
-    armmp-lpae kirkwood loongson-3 marvell octeon orion5x powerpc64le \
-    rt-686-pae rt-amd64 s390x"
diff --git a/meta-isar/conf/distro/debian-stretch.list b/meta-isar/conf/distro/debian-stretch.list
new file mode 100644
index 0000000..bfa5cf9
--- /dev/null
+++ b/meta-isar/conf/distro/debian-stretch.list
@@ -0,0 +1,3 @@
+deb	http://ftp.de.debian.org/debian stretch	main contrib non-free
+deb	http://ftp.de.debian.org/debian stretch-updates	main contrib non-free
+deb	http://security.debian.org	stretch/updates	main contrib non-free
diff --git a/meta-isar/conf/distro/debian-wheezy.conf b/meta-isar/conf/distro/debian-wheezy.conf
index f5ab6eb..2108f9b 100644
--- a/meta-isar/conf/distro/debian-wheezy.conf
+++ b/meta-isar/conf/distro/debian-wheezy.conf
@@ -1,14 +1,17 @@
-# Debian wheezy multistrap configuration
+# Debian wheezy distribution configuration
 #
 # This software is a part of ISAR.
 # Copyright (C) 2015-2016 ilbers GmbH
 
+DISTRO_APT_SOURCES += "conf/distro/debian-wheezy.list"
+DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
+DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile vexpress \
+    iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x"
+
+# No longer needed:
 DISTRO_SUITE ?= "wheezy"
 DISTRO_COMPONENTS ?= "main contrib non-free"
 DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
 DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
-DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
 DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
 DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
-DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile vexpress \
-    iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x"
diff --git a/meta-isar/conf/distro/debian-wheezy.list b/meta-isar/conf/distro/debian-wheezy.list
new file mode 100644
index 0000000..0bfa98e
--- /dev/null
+++ b/meta-isar/conf/distro/debian-wheezy.list
@@ -0,0 +1,3 @@
+deb	http://ftp.de.debian.org/debian wheezy	main contrib non-free
+deb	http://ftp.de.debian.org/debian wheezy-updates	main contrib non-free
+deb	http://security.debian.org	wheezy/updates	main contrib non-free
diff --git a/meta-isar/conf/distro/raspbian-jessie.conf b/meta-isar/conf/distro/raspbian-jessie.conf
index c2a867d..57d0e7d 100644
--- a/meta-isar/conf/distro/raspbian-jessie.conf
+++ b/meta-isar/conf/distro/raspbian-jessie.conf
@@ -1,13 +1,17 @@
-# Raspbian stable multistrap configuration
+# Raspbian stable distribution configuration
 #
 # This software is a part of ISAR.
 # Copyright (C) 2015-2016 ilbers GmbH
 
+DISTRO_APT_SOURCES += "conf/distro/raspbian-jessie.list"
+DISTRO_APT_KEYS += "https://archive.raspbian.org/raspbian.public.key;sha256sum=ca59cd4f2bcbc3a1d41ba6815a02a8dc5c175467a59bd87edeac458f4a5345de"
+DISTRO_CONFIG_SCRIPT?= "raspbian-configscript.sh"
+DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv"
+
+# No longer needed:
 DISTRO_SUITE ?= "jessie"
 DISTRO_COMPONENTS ?= "main contrib non-free firmware"
 DISTRO_APT_SOURCE ?= "http://archive.raspbian.org/raspbian"
 DISTRO_APT_SOURCE_SEC ?= ""
-DISTRO_CONFIG_SCRIPT ?= "raspbian-configscript.sh"
 DISTRO_MULTICONF_BOOTSTRAP ?= "base"
 DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
-DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv"
diff --git a/meta-isar/conf/distro/raspbian-jessie.list b/meta-isar/conf/distro/raspbian-jessie.list
new file mode 100644
index 0000000..792f9c6
--- /dev/null
+++ b/meta-isar/conf/distro/raspbian-jessie.list
@@ -0,0 +1 @@
+deb	http://archive.raspbian.org/raspbian	jessie	main contrib non-free firmware
diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt.conf b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf
new file mode 100644
index 0000000..a62feb1
--- /dev/null
+++ b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf
@@ -0,0 +1,2 @@
+APT::Install-Recommends "0";
+APT::Install-Suggests "0";
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
new file mode 100644
index 0000000..a38dd88
--- /dev/null
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
@@ -0,0 +1,206 @@
+# Minimal debian root file system
+#
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2018
+#
+# SPDX-License-Identifier: MIT
+
+Description = "Minimal debian root file system"
+
+LICENSE = "gpl-2.0"
+LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
+FILESPATH_prepend := "${THISDIR}/files:"
+SRC_URI = "file://isar-apt.conf"
+PV = "1.0"
+
+WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+DEBOOTSTRAP ?= ""
+ROOTFSDIR = "${WORKDIR}/rootfs"
+APTPREFS = "${WORKDIR}/apt-preferences"
+APTSRCS = "${WORKDIR}/apt-sources"
+APTKEYFILES = ""
+APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
+DEBOOTSTRAP_KEYRING = ""
+
+python () {
+    from urllib.parse import urlparse
+
+    debootstrap = d.getVar("DEBOOTSTRAP", True)
+    if not debootstrap:
+        target = d.getVar("DISTRO_ARCH", True)
+        machine = os.uname()[4]
+        m = {
+            "x86_64": ["i386", "amd64"],
+            "x86": ["i386"],
+            }
+        if machine not in m or target not in m[machine]:
+            debootstrap = "qemu-debootstrap"
+        else:
+            debootstrap = "debootstrap"
+        d.setVar("DEBOOTSTRAP", debootstrap)
+
+    distro_apt_keys = d.getVar("DISTRO_APT_KEYS", False)
+    if distro_apt_keys:
+        d.setVar("DEBOOTSTRAP_KEYRING", "--keyring ${APTKEYRING}")
+        for key in distro_apt_keys.split():
+            url = urlparse(key)
+            filename = os.path.basename(url.path)
+            d.appendVar("SRC_URI", " %s" % key)
+            d.appendVar("APTKEYFILES", " %s" % filename)
+}
+
+def aggregate_files(d, file_list, file_out):
+    import shutil
+
+    with open(file_out, "wb") as out_fd:
+        for entry in file_list:
+            entry_real = bb.parse.resolve_file(entry, d)
+            with open(entry_real, "rb") as in_fd:
+                 shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
+            out_fd.write("\n".encode())
+
+def parse_aptsources_list_line(source_list_line):
+    import re
+
+    s = source_list_line.strip()
+
+    if s.startswith("#"):
+        return None
+
+    type, s = re.split("\s+", s, maxsplit=1)
+    if type not in ["deb", "deb-src"]:
+        return None
+
+    options = ""
+    options_match = re.match("\[\s*(\S+=\S+(?=\s))*\s*(\S+=\S+)\s*\]\s+", s)
+    if options_match:
+        options = options_match.group(0).strip()
+        s = s[options_match.end():]
+
+    source, s = re.split("\s+", s, maxsplit=1)
+
+    suite, s = re.split("\s+", s, maxsplit=1)
+
+    components = " ".join(s.split())
+
+    return type, options, source, suite, components
+
+def get_distro_primary_source_entry(d):
+    apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+    for entry in apt_sources_list:
+        entry_real = bb.parse.resolve_file(entry, d)
+        with open(entry_real, "r") as in_fd:
+            for line in in_fd:
+                parsed = parse_aptsources_list_line(line)
+                if parsed:
+                    type, _, source, suite, components = parsed
+                    if type == "deb":
+                        return source, suite, components
+    return "", "", ""
+
+def get_distro_source(d):
+    return get_distro_primary_source_entry(d)[0]
+
+def get_distro_suite(d):
+    return get_distro_primary_source_entry(d)[1]
+
+def get_distro_components_argument(d):
+    components = get_distro_primary_source_entry(d)[2]
+    if components and components.strip():
+        return "--components=%s" % ",".join(components.split())
+    else:
+        return ""
+
+do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_generate_keyring[dirs] = "${WORKDIR}"
+do_generate_keyring[vardeps] += "DISTRO_APT_KEYS"
+do_generate_keyring() {
+    if [ -n "${@d.getVar("APTKEYFILES", True) or ""}" ]; then
+        for keyfile in ${@d.getVar("APTKEYFILES", True)}; do
+           gpg --no-default-keyring --keyring "${APTKEYRING}" \
+               --homedir "${WORKDIR}" --import "$keyfile"
+        done
+    fi
+}
+addtask generate_keyring before do_build after do_unpack
+
+do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_apt_config_prepare[vardeps] += "\
+                                   APTPREFS \
+                                   DISTRO_APT_PREFERENCES \
+                                   DEBDISTRONAME \
+                                   APTSRCS \
+                                   DISTRO_APT_SOURCES \
+                                  "
+python do_apt_config_prepare() {
+    apt_preferences_out = d.getVar("APTPREFS", True)
+    apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
+                           ).split()
+    aggregate_files(d, apt_preferences_list, apt_preferences_out)
+
+    apt_sources_out = d.getVar("APTSRCS", True)
+    apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+
+    aggregate_files(d, apt_sources_list, apt_sources_out)
+}
+addtask apt_config_prepare before do_build after do_generate_keyring
+
+do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
+do_bootstrap() {
+    if [ -e "${ROOTFSDIR}" ]; then
+       sudo umount -l "${ROOTFSDIR}/dev" || true
+       sudo umount -l "${ROOTFSDIR}/proc" || true
+       sudo rm -rf "${ROOTFSDIR}"
+    fi
+    E="${@bb.utils.export_proxies(d)}"
+    sudo -E "${DEBOOTSTRAP}" --verbose \
+                             --variant minbase \
+                             --arch "${DISTRO_ARCH}" \
+                             ${@get_distro_components_argument(d)} \
+                             ${DEBOOTSTRAP_KEYRING} \
+                             "${@get_distro_suite(d)}" \
+                             "${ROOTFSDIR}" \
+                             "${@get_distro_source(d)}"
+}
+addtask bootstrap before do_build after do_apt_config_prepare
+
+do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_apt_config_install() {
+    sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+    sudo install -v -m644 "${APTPREFS}" \
+                          "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+    sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+    sudo install -v -m644 "${APTSRCS}" \
+                          "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+    sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+    sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
+    sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
+                          "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
+}
+addtask apt_config_install before do_build after do_bootstrap
+
+do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_apt_update() {
+    sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
+    sudo mount -t proc none ${ROOTFSDIR}/proc
+
+    E="${@bb.utils.export_proxies(d)}"
+    export DEBIAN_FRONTEND=noninteractive
+    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
+    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+                                      -o Debug::pkgProblemResolver=yes
+}
+addtask apt_update before do_build after do_apt_config_install
+
+do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
+do_deploy() {
+    ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
+}
+addtask deploy before do_build after do_apt_update
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+     rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
+}
-- 
2.16.3