[RFC PATCH 1/1] RFC-PATCH: partial implementation for support of repos from other layers

[claudius.heine.ext@siemens.com]

From: Claudius Heine <ch@denx.de>

This patch changes the implementation of isar-image-base to support
the definition of additional repositories from other layers, by addding
apt source list compatible files to the 'DEBIAN_APT_SOURCES' variable.

This also implement the generation of the multistrap.conf file from
within the recipe.

Signed-off-by: Claudius Heine <ch@denx.de>
---
 meta-isar/conf/distro/debian-stretch.conf          |   6 +-
 meta-isar/conf/distro/stretch.list                 |   3 +
 .../recipes-core/images/files/multistrap.conf.in   |  38 ------
 meta-isar/recipes-core/images/isar-image-base.bb   | 134 +++++++++++++++++----
 4 files changed, 117 insertions(+), 64 deletions(-)
 create mode 100644 meta-isar/conf/distro/stretch.list
 delete mode 100644 meta-isar/recipes-core/images/files/multistrap.conf.in

diff --git a/meta-isar/conf/distro/debian-stretch.conf b/meta-isar/conf/distro/debian-stretch.conf
index 74177fd..083eb86 100644
--- a/meta-isar/conf/distro/debian-stretch.conf
+++ b/meta-isar/conf/distro/debian-stretch.conf
@@ -1,10 +1,14 @@
 # This software is a part of ISAR.
 # Copyright (C) 2017 ilbers GmbH
 
+DEBIAN_APT_SOURCES += "conf/distro/stretch.list"
+DEBIAN_APT_PREFERENCES ??= ""
+DISTRO_CONFIG_SCRIPT ??= "debian-configscript.sh"
+
+# No longer needed, after buildchroot is converted:
 DISTRO_SUITE ?= "stretch"
 DISTRO_COMPONENTS ?= "main contrib non-free"
 DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
 DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
-DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
 DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
 DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
diff --git a/meta-isar/conf/distro/stretch.list b/meta-isar/conf/distro/stretch.list
new file mode 100644
index 0000000..a5ed5db
--- /dev/null
+++ b/meta-isar/conf/distro/stretch.list
@@ -0,0 +1,3 @@
+deb	http://ftp.de.debian.org/debian	stretch		main contrib non-free
+deb	http://ftp.de.debian.org/debian stretch-updates	main contrib non-free
+deb	http://security.debian.org	stretch/updates	main contrib non-free
diff --git a/meta-isar/recipes-core/images/files/multistrap.conf.in b/meta-isar/recipes-core/images/files/multistrap.conf.in
deleted file mode 100644
index 432b6af..0000000
--- a/meta-isar/recipes-core/images/files/multistrap.conf.in
+++ /dev/null
@@ -1,38 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) 2015-2017 ilbers GmbH
-
-[General]
-noauth=true
-unpack=true
-ignorenativearch=true
-bootstrap=##DISTRO_MULTICONF_BOOTSTRAP## Isar
-aptsources=##DISTRO_MULTICONF_APTSOURCES##
-configscript=##CONFIG_SCRIPT##
-setupscript=##SETUP_SCRIPT##
-hookdir=##DIR_HOOKS##
-
-[base]
-source=##DISTRO_APT_SOURCE##
-suite=##DISTRO_SUITE##
-components=##DISTRO_COMPONENTS##
-packages=##IMAGE_PREINSTALL##
-omitdebsrc=true
-
-[updates]
-source=##DISTRO_APT_SOURCE##
-suite=##DISTRO_SUITE##-updates
-components=##DISTRO_COMPONENTS##
-omitdebsrc=true
-
-[security]
-source=##DISTRO_APT_SOURCE_SEC##
-suite=##DISTRO_SUITE##/updates
-components=##DISTRO_COMPONENTS##
-omitdebsrc=true
-
-[Isar]
-packages=##IMAGE_INSTALL##
-source=##DEPLOY_DIR_APT##
-suite=##ISAR_DISTRO_SUITE##
-components=main
-omitdebsrc=true
diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
index e359ac3..655dbb2 100644
--- a/meta-isar/recipes-core/images/isar-image-base.bb
+++ b/meta-isar/recipes-core/images/isar-image-base.bb
@@ -9,8 +9,7 @@ LICENSE = "gpl-2.0"
 LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
 
 FILESPATH =. "${LAYERDIR_isar}/recipes-core/images/files:"
-SRC_URI = "file://multistrap.conf.in \
-	   file://${DISTRO_CONFIG_SCRIPT} \
+SRC_URI = "file://${DISTRO_CONFIG_SCRIPT} \
 	   file://setup.sh \
 	   file://download_dev-random"
 
@@ -26,34 +25,119 @@ IMAGE_PREINSTALL += "apt \
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
 
-do_rootfs[dirs] = "${WORKDIR}/hooks_multistrap"
-
-do_rootfs() {
-    E="${@ bb.utils.export_proxies(d)}"
+do_multistrap_prepare[dirs] = "${WORKDIR}/hooks_multistrap"
 
+do_multistrap_prepare() {
     chmod +x "${WORKDIR}/${DISTRO_CONFIG_SCRIPT}"
     chmod +x "${WORKDIR}/setup.sh"
     install -m 755 "${WORKDIR}/download_dev-random" "${WORKDIR}/hooks_multistrap/"
+}
+addtask multistrap_prepare before do_multistrap_config after do_unpack
+
+python do_multistrap_config() {
+    import shutil
+    import configparser as cp
+    import textwrap
+
+    workdir = d.getVar("WORKDIR", True)
+    workdir_rel = os.path.relpath(workdir)
+
+    multistrap_conf_file = os.path.join(workdir, "multistrap.conf")
+
+    apt_sources_list = d.getVar("DEBIAN_APT_SOURCES", True).split()
+
+    apt_preferences_list = d.getVar("DEBIAN_APT_PREFERENCES", True).split()
+    apt_preferences_out = os.path.join(workdir_rel, "apt-preferences")
+
+    image_preinstall = d.getVar("IMAGE_PREINSTALL", True).split()
+    image_install = d.getVar("IMAGE_INSTALL", True).split()
+
+    repos = []
+
+    msconf = cp.RawConfigParser()
+
+    # Set general multistrap configuration:
+    msconf.add_section("General")
+    msconf.set("General", "noauth", "true")
+    msconf.set("General", "unpack", "true")
+    msconf.set("General", "ignorenativearch", "true")
+    msconf.set("General", "aptpreferences", "./" + apt_preferences_out)
+    msconf.set("General", "configscript", "./" +
+               os.path.join(workdir_rel,
+                            d.getVar("DISTRO_CONFIG_SCRIPT", True)))
+    msconf.set("General", "setupscript", "./" +
+               os.path.join(workdir_rel,
+                            "setup.sh"))
+    msconf.set("General", "hookdir", "./" +
+               os.path.join(workdir_rel,
+                            "hooks_multistrap"))
+
+    # Set repository in multistrap configuration:
+    repo_counter = 0
+    for aptsrc in apt_sources_list:
+        aptsrc_real = bb.parse.resolve_file(aptsrc, d)
+        with open(aptsrc_real, "r") as in_fd:
+            for line in in_fd:
+                entry = line.split()
+
+                # Only care about 'deb' repos for now:
+                if entry[0] != "deb":
+                    continue
+
+                # Generate on unique name for the section
+                repo_name = "repo{}".format(repo_counter)
+                repos.append(repo_name)
+
+                source = entry[1]
+                suite = entry[2]
+                components = entry[3:]
+
+                msconf.add_section(repo_name)
+                msconf.set(repo_name, "source", source)
+                msconf.set(repo_name, "suite", suite)
+                msconf.set(repo_name, "components", " ".join(components))
+                msconf.set(repo_name, "omitdebsrc", "true")
+
+                # The first repo gets assigned all IMAGE_PREINSTALL
+                # packages:
+                if image_preinstall:
+                   msconf.set(repo_name, "packages",
+                              " ".join(image_preinstall))
+                   image_preinstall = []
+
+                repo_counter += 1
+
+    msconf.add_section("isar-generated")
+    msconf.set("isar-generated", "packages", " ".join(image_install))
+    msconf.set("isar-generated", "source", "copy:///{}/{}".format(
+               d.getVar("DEPLOY_DIR_APT", True), d.getVar("DISTRO", True)))
+    msconf.set("isar-generated", "suite", d.getVar("DEBDISTRONAME", True))
+    msconf.set("isar-generated", "components", "main")
+    msconf.set("isar-generated", "omitdebsrc", "true")
+
+    # Generate apt preference file:
+    with open(apt_preferences_out, "wb") as out_fd:
+        for aptpref in apt_preferences_list:
+            artpref_real = bb.parse.resolve_file(aptpref, d)
+            with open(aptpref_real, "rb") as in_fd:
+                 shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
+            out_fd.write("\n".encode())
+        out_fd.write(textwrap.dedent("""\
+                     Package: *
+                     Pin: release n={}
+                     Pin-Priority: 1000
+                     """.format(d.getVar("DEBDISTRONAME", True))).encode())
+
+    msconf.set("General", "bootstrap", " ".join(repos))
+    msconf.set("General", "aptsources", " ".join(repos + ["isar-generated"]))
+
+    with open(multistrap_conf_file, "w") as out_fd:
+        msconf.write(out_fd)
+}
+addtask multistrap_config after do_multistrap_prepare before do_rootfs
 
-    # Multistrap accepts only relative path in configuration files, so get it:
-    cd ${TOPDIR}
-    WORKDIR_REL=${@ os.path.relpath(d.getVar("WORKDIR", True))}
-
-    # Adjust multistrap config
-    sed -e 's|##IMAGE_PREINSTALL##|${IMAGE_PREINSTALL}|g' \
-        -e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
-        -e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
-        -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
-        -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
-        -e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
-        -e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
-        -e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/${DISTRO_CONFIG_SCRIPT}|g' \
-        -e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
-        -e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
-        -e 's|##IMAGE_INSTALL##|${IMAGE_INSTALL}|g' \
-        -e 's|##DEPLOY_DIR_APT##|copy:///${DEPLOY_DIR_APT}/${DISTRO}|g' \
-        -e 's|##ISAR_DISTRO_SUITE##|${DEBDISTRONAME}|g' \
-           "${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
+do_rootfs() {
+    E="${@ bb.utils.export_proxies(d)}"
 
     [ ! -d ${IMAGE_ROOTFS}/proc ] && sudo install -d -o 0 -g 0 -m 555 ${IMAGE_ROOTFS}/proc
     sudo mount -t proc none ${IMAGE_ROOTFS}/proc
-- 
2.15.1