[RFC v2][PATCH 3/3] base-apt: Introduce fetching upstream apt

[Alexander Smirnov <asmirnov@ilbers.de>]

This patch introduces mechanism how to fetch deb packages from dedicated
upstream apt repos and store them localy. Local repository is called
'base-apt' and it will be used to generate buildchroot and image root
filesystems. Using 'base-apt' will guarantee build reproducibility between
builds.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
---
 meta/conf/isar-bitbake.conf                        |  2 +
 meta/recipes-devtools/base-apt/base-apt.bb         | 95 ++++++++++++++++++++++
 .../base-apt/files/distributions.in                |  3 +
 .../base-apt/files/multistrap.conf.in              | 28 +++++++
 meta/recipes-devtools/buildchroot/buildchroot.bb   |  8 ++
 5 files changed, 136 insertions(+)
 create mode 100644 meta/recipes-devtools/base-apt/base-apt.bb
 create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
 create mode 100644 meta/recipes-devtools/base-apt/files/multistrap.conf.in

diff --git a/meta/conf/isar-bitbake.conf b/meta/conf/isar-bitbake.conf
index 5a26743..df54399 100644
--- a/meta/conf/isar-bitbake.conf
+++ b/meta/conf/isar-bitbake.conf
@@ -23,6 +23,8 @@ DEPLOY_DIR_DEB = "${TMPDIR}/deploy/deb/${MACHINE}"
 SSTATE_DIR ?= "${TMPDIR}/sstate-cache"
 BUILDCHROOT_DIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/buildchroot/rootfs"
 
+BASE_APT_DIR ?= "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/base-apt"
+
 # Setup our default hash policy
 BB_SIGNATURE_HANDLER ?= "noop"
 
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
new file mode 100644
index 0000000..e05ea61
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -0,0 +1,95 @@
+# Caching upstream apt repository to local one.
+#
+# This software is a part of ISAR.
+# Copyright (C) 2015-2017 ilbers GmbH
+
+DESCRIPTION = "Upstream apt caching"
+
+LICENSE = "gpl-2.0"
+LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
+
+FILESPATH =. "${LAYERDIR_core}/recipes-devtools/base-apt/files:"
+SRC_URI = "file://distributions.in \
+           file://multistrap.conf.in \
+          "
+
+BASE_PREINSTALL ?= ""
+
+WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+
+do_fetch_debs[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+
+do_fetch_debs() {
+    for package in `ls ${WORKDIR}/deps/*.depends`; do
+        DEPS=$(perl -ne 'next if /^#/; $p=(s/,|\n|\([^)]+\)|\[[^]]+\]//mg); print if $p' < $package)
+        PACKAGES="$PACKAGES $DEPS"
+    done
+
+    for package in `ls ${WORKDIR}/deps/*.preinst`; do
+        DEPS=$(cat $package | xargs)
+        PACKAGES="$PACKAGES $DEPS"
+    done
+
+    PACKAGES="$PACKAGES $BASE_PREINSTALL"
+
+    # Adjust multistrap config
+    sed -e 's|##BASE_PREINSTALL##|'"$PACKAGES"'|g' \
+        -e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
+        -e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
+        -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
+        -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
+        -e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
+        -e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
+        -e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/configscript.sh|g' \
+        -e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
+        -e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
+           "${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
+
+    # Fetch deb packages
+    sudo -E multistrap \
+        -a ${DISTRO_ARCH} \
+        -d "${WORKDIR}/download" \
+        -f "${WORKDIR}/multistrap.conf" \
+        > ${WORKDIR}/multistrap.log 2>&1
+}
+
+addtask fetch_debs after do_unpack before do_build
+
+BASE_APT_CONF_DIR = "${BASE_APT_DIR}/apt/conf"
+do_build[dirs] = "${BASE_APT_CONF_DIR}"
+do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+
+do_build() {
+    if [ ! -e "${BASE_APT_CONF_DIR}/distributions" ]; then
+        sed -e "s#{DISTRO_NAME}#"${DISTRO_SUITE}"#g" \
+            ${WORKDIR}/distributions.in > ${BASE_APT_CONF_DIR}/distributions
+    fi
+
+    # Create reprepro cache
+    if [ ! -d "${BASE_APT_DIR}/apt" ]; then
+        reprepro -b ${BASE_APT_DIR}/apt \
+                 --dbdir ${BASE_APT_DIR}/db \
+                 export ${DISTRO_SUITE}
+    fi
+
+    # Process all the packages fetched by multistrap
+    for deb in $(ls ${WORKDIR}/download/var/cache/apt/archives/*.deb);
+    do
+        pn=$(dpkg-deb -I $deb | grep 'Package:' | cut -d ' ' -f 3)
+        pv=$(dpkg-deb -I $deb | grep 'Version:' | cut -d ' ' -f 3)
+        line=$(cat ${WORKDIR}/multistrap.log | grep -E '^(Get:)' | grep " $pn ")
+        url=$(echo $line | cut -d ' ' -f 2)
+        component=$(basename $(echo $line | cut -d ' ' -f 3))
+
+        # Store download history
+        echo $pn $pv $component $url >> ${WORKDIR}/deb.list
+
+        reprepro -b ${BASE_APT_DIR}/apt \
+                 --dbdir ${BASE_APT_DIR}/db \
+                 -C $component \
+                 includedeb ${DISTRO_SUITE} \
+                 $deb
+    done
+
+    sudo rm -rf ${WORKDIR}/download
+}
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
new file mode 100644
index 0000000..44e9513
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/files/distributions.in
@@ -0,0 +1,3 @@
+Codename: {DISTRO_NAME}
+Architectures: i386 armhf amd64 source
+Components: main contrib non-free firmware
diff --git a/meta/recipes-devtools/base-apt/files/multistrap.conf.in b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
new file mode 100644
index 0000000..27bf985
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
@@ -0,0 +1,28 @@
+# This software is a part of ISAR.
+# Copyright (C) 2015-2017 ilbers GmbH
+
+[General]
+noauth=true
+unpack=false
+ignorenativearch=true
+bootstrap=##DISTRO_MULTICONF_BOOTSTRAP##
+aptsources=##DISTRO_MULTICONF_APTSOURCES##
+
+[base]
+source=##DISTRO_APT_SOURCE##
+suite=##DISTRO_SUITE##
+components=##DISTRO_COMPONENTS##
+packages=##BASE_PREINSTALL##
+omitdebsrc=true
+
+[updates]
+source=##DISTRO_APT_SOURCE##
+suite=##DISTRO_SUITE##-updates
+components=##DISTRO_COMPONENTS##
+omitdebsrc=true
+
+[security]
+source=##DISTRO_APT_SOURCE_SEC##
+suite=##DISTRO_SUITE##/updates
+components=##DISTRO_COMPONENTS##
+omitdebsrc=true
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index da18231..a551e50 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -33,6 +33,14 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
 
+python __anonymous() {
+    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
+    if rep == "0":
+        return
+
+    d.setVarFlag("do_build", "depends", "base-apt:do_build")
+}
+
 do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_build[dirs] = "${WORKDIR}/hooks_multistrap"
 
-- 
2.1.4