[PATCH 1/5] base: add populate_repo task to include distro packages to the repo

[<Cedric_Hombourger@mentor.com>]

From: Cedric Hombourger <Cedric_Hombourger@mentor.com>

A package repository is already created for custom packages (built by
Isar). Provide a populate_repo task to also include any packages
included into images built by Isar. This mechanism will add both binary
and source packages. The augmented repository may be used later for
reproducible builds (technically, packages pulled for the buildchroots
we build should also be added; this will be the subject of another
changeset).

Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
---
 .../recipes-core/images/files/multistrap.conf.in   |  6 +-
 meta-isar/recipes-core/images/isar-image-base.bb   | 78 ++++++++++++++++++++++
 2 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/meta-isar/recipes-core/images/files/multistrap.conf.in b/meta-isar/recipes-core/images/files/multistrap.conf.in
index 432b6af..8e4e276 100644
--- a/meta-isar/recipes-core/images/files/multistrap.conf.in
+++ b/meta-isar/recipes-core/images/files/multistrap.conf.in
@@ -16,19 +16,19 @@ source=##DISTRO_APT_SOURCE##
 suite=##DISTRO_SUITE##
 components=##DISTRO_COMPONENTS##
 packages=##IMAGE_PREINSTALL##
-omitdebsrc=true
+omitdebsrc=false
 
 [updates]
 source=##DISTRO_APT_SOURCE##
 suite=##DISTRO_SUITE##-updates
 components=##DISTRO_COMPONENTS##
-omitdebsrc=true
+omitdebsrc=false
 
 [security]
 source=##DISTRO_APT_SOURCE_SEC##
 suite=##DISTRO_SUITE##/updates
 components=##DISTRO_COMPONENTS##
-omitdebsrc=true
+omitdebsrc=false
 
 [Isar]
 packages=##IMAGE_INSTALL##
diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
index dea2cdb..6ce3d6d 100644
--- a/meta-isar/recipes-core/images/isar-image-base.bb
+++ b/meta-isar/recipes-core/images/isar-image-base.bb
@@ -52,6 +52,79 @@ do_rootfs_populate_dev() {
     _mknod_cdev 666 ptmx    5 2
 }
 
+do_populate_repo() {
+
+    APT_CACHEDIR=${IMAGE_ROOTFS}/var/cache/apt # Home of the apt cache (for the target image)
+    APT_DEBDIR=${APT_CACHEDIR}/archives        # Downloaded packages can be found there
+    APT_SRCDIR=${WORKDIR}/apt-sources          # Put source packages outside of the rootfs
+
+    sudo mkdir -p ${APT_SRCDIR}
+
+    apt_status_file=$(mktemp)
+    for p in ${DEPLOY_DIR_DEB}/${PN}/*.deb; do
+        # Only process packages that were NOT built by Isar (already added to the repo)
+        deb=$(basename ${p})
+        if [ ! -f ${DEPLOY_DIR_DEB}/${deb} ]; then
+            # Extract package name
+            pkg=$(dpkg-deb -I ${p}|grep -e '^ Package:'|sed -e 's,^ Package: ,,g')
+
+            # Extract package section and priority so we can tell rerepro where to place this package
+            sect=$(dpkg-deb -I ${p}|grep -e '^ Section:'|sed -e 's,^ Section: ,,g')
+            prio=$(dpkg-deb -I ${p}|grep -e '^ Priority:'|sed -e 's,^ Priority: ,,g')
+
+            # Determine name of the source package (same as the binary package if not specified)
+            src=$(dpkg-deb -I ${p}|grep -e '^ Source:'|sed -e 's,^ Source: ,,g'|awk '{ print $1; }')
+            src=${src:-${pkg}}
+
+            # Add the binary package to the repo
+            call_reprepro -b ${DEPLOY_DIR_APT}/${DISTRO}   \
+                --dbdir ${DEPLOY_DIR_DB}/${DISTRO}         \
+                -C main -S ${sect} -P ${prio}              \
+                includedeb ${DEBDISTRONAME} ${p}
+
+            # Get source package and add it to the repo
+            # Notes:
+            #   - multistrap --source-dir is failing to fetch sources hence this manual process
+            #     (the culprit appears to be the following apt-get option: APT::Default-Release='*')
+            #   - the apt-get command-line was copied from multistrap and tweaked for sources
+            #     (a separate/temporary apt status file is used to avoid polluting the existing one
+            #      with source transactions)
+            #   - as we assume sources were already fetched if the directory exists, quietly
+            #     attempt to delete it first; if sources are indeed there, it will fetch. If
+            #     they aren't, the directory will be deleted and we will have another go at it
+            rmdir ${APT_SRCDIR}/${src} 2>/dev/null || true
+            if [ ! -d "${APT_SRCDIR}/${src}" ]; then
+                sudo mkdir -p ${APT_SRCDIR}/${src}
+                cd ${APT_SRCDIR}/${src}
+                bbnote "fetching source package '${src}' (binary: ${pkg})"
+                sudo -E apt-get \
+                    -o Dir::Etc::TrustedParts=${IMAGE_ROOTFS}/etc/apt/trusted.gpg.d \
+                    -o Dir::Etc::Trusted=${IMAGE_ROOTFS}/etc/apt/trusted.gpg \
+                    -o Apt::Get::AllowUnauthenticated=true \
+                    -o Apt::Get::Download-Only=true \
+                    -o Apt::Install-Recommends=false \
+                    -o Dir=${IMAGE_ROOTFS} \
+                    -o Dir::Etc=${IMAGE_ROOTFS}/etc/apt/ \
+                    -o Dir::Etc::Parts=${IMAGE_ROOTFS}/etc/apt/apt.conf.d/ \
+                    -o Dir::Etc::PreferencesParts=${IMAGE_ROOTFS}/etc/apt/preferences.d/ \
+                    -o Dir::State=${IMAGE_ROOTFS}/var/lib/apt/ \
+                    -o Dir::State::Status=${apt_status_file} \
+                    -o Dir::Cache=${IMAGE_ROOTFS}/var/cache/apt/ -y source ${src}
+                call_reprepro -b ${DEPLOY_DIR_APT}/${DISTRO}   \
+                    --dbdir ${DEPLOY_DIR_DB}/${DISTRO}         \
+                    -C main -S ${sect} -P ${prio}              \
+                    includedsc ${DEBDISTRONAME}                \
+                    ${APT_SRCDIR}/${src}/*.dsc
+            else
+                bbnote "already fetched source package '${src}' (binary: ${pkg})"
+            fi
+        fi
+    done
+    rm -f ${apt_status_file}
+}
+
+addtask populate_repo
+
 do_rootfs() {
     E="${@ bb.utils.export_proxies(d)}"
 
@@ -94,6 +167,11 @@ do_rootfs() {
     # because of proxy settings
     sudo -E multistrap -a ${DISTRO_ARCH} -d "${IMAGE_ROOTFS}" -f "${WORKDIR}/multistrap.conf"
 
+    # Copy packages before they vanish when the root file-systems get configured
+    # (apt purge is being run)
+    install -m 755 -d ${DEPLOY_DIR_DEB}/${PN}
+    sudo cp -l ${IMAGE_ROOTFS}/var/cache/apt/archives/*.deb ${DEPLOY_DIR_DEB}/${PN}/
+
     # Configure root filesystem
     sudo chroot ${IMAGE_ROOTFS} /${DISTRO_CONFIG_SCRIPT} ${MACHINE_SERIAL} ${BAUDRATE_TTY} \
         ${ROOTFS_DEV}
-- 
2.11.0