[PATCH v5 08/12] base-apt: Predownload packages to base-apt before install

[Uladzimir Bely <ubely@ilbers.de>]

This patch uses debrepo script to predownload packages to base-apt
repository before they are installed in buildchroot/rootfs.

Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
 meta/classes/dpkg-base.bbclass               | 36 ++++++++++++++++++--
 meta/classes/dpkg.bbclass                    |  8 +++++
 meta/classes/image-locales-extension.bbclass |  5 +++
 meta/classes/image-tools-extension.bbclass   |  9 +++++
 meta/classes/rootfs.bbclass                  |  4 +++
 5 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index c3e47a69..faf2ec77 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -23,6 +23,8 @@ python __anonymous() {
         d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True))
 }
 
+inherit debrepo
+
 DEPENDS ?= ""
 RPROVIDES ?= "${PROVIDES}"
 
@@ -126,10 +128,25 @@ do_apt_fetch() {
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap 'schroot_cleanup' EXIT
 
+    # Begin chroot session
+    session_id=$(schroot -b -c ${SBUILD_CHROOT})
+
+    debrepo_add_packages --srcmode "${DEBREPO_TARGET_DIR}" "${SRC_APT}"
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+        schroot -d / -r -c ${session_id} -u root -- \
+            sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+        "
+    fi
+
     for uri in "${SRC_APT}"; do
-        schroot -d / -c ${SBUILD_CHROOT} -- \
+        schroot -d / -r -c ${session_id} -- \
             sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}"
     done
+
+    # End chroot session
+    schroot -e -c ${session_id}
+
     schroot_delete_configs
 }
 
@@ -139,6 +156,8 @@ do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 
 # Add dependency from the correct schroot: host or target
 do_apt_fetch[depends] += "${SCHROOT_DEP}"
+# Debrepo context is created by target bootstrap, need this dependency too
+do_apt_fetch[depends] += "isar-bootstrap-target:do_bootstrap"
 
 do_apt_unpack() {
     rm -rf ${S}
@@ -150,8 +169,18 @@ do_apt_unpack() {
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap 'schroot_cleanup' EXIT
 
+    # Begin chroot session
+    session_id=$(schroot -b -c ${SBUILD_CHROOT})
+
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+        schroot -d / -r -c ${session_id} -u root -- \
+            sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+        "
+    fi
+
     for uri in "${SRC_APT}"; do
-        schroot -d / -c ${SBUILD_CHROOT} -- \
+        schroot -d / -r -c ${session_id} -- \
             sh -c ' \
                 set -e
                 dscfile="$(apt-get -y -qq --print-uris --only-source source "${2}" | cut -d " " -f2 | grep -E "*.dsc")"
@@ -160,6 +189,9 @@ do_apt_unpack() {
                 dpkg-source -x "${dscfile}" "${PPS}"' \
                     my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}"
     done
+
+    # End chroot session
+    schroot -e -c ${session_id}
     schroot_delete_configs
 }
 do_apt_unpack[network] = "${TASK_USE_SUDO}"
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 5c8996d6..ea26b530 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -32,6 +32,7 @@ dpkg_runbuild() {
     export PARALLEL_MAKE="${PARALLEL_MAKE}"
 
     rm -f ${SBUILD_CONFIG}
+    debrepo_handle_controlfile "${WORKDIR}/${PPS}/debian/control"
 
     env | while read -r line; do
         # Filter the same lines
@@ -101,6 +102,11 @@ dpkg_runbuild() {
     sh -c "cd ${WORKDIR}; dpkg-source -q -b ${PPS}"
     DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -print)
 
+    locked_update_cmd=":"
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        locked_update_cmd="flock -x /base-apt/repo.lock -c 'apt-get -y update'"
+    fi
+
     sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \
         --host=${PACKAGE_ARCH} --build=${BUILD_HOST_ARCH} ${profiles} \
         --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \
@@ -109,8 +115,10 @@ dpkg_runbuild() {
         --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
         --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \
+        --chroot-setup-commands="${locked_update_cmd}" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
         --finished-build-commands="cp -Ln --no-preserve=owner ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \
+        --finished-build-commands="mkdir -p ${ext_root}" \
         --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
         --debbuildopts="--source-option=-I" \
         --build-dir=${WORKDIR} --dist="isar" ${DSC_FILE}
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index b48bcec8..aa4874c1 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -6,6 +6,8 @@
 # This class extends the image.bbclass for setting locales and purging unneeded
 # ones.
 
+inherit debrepo
+
 LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
                en_US ISO-8859-1\n"
 LOCALE_DEFAULT ?= "en_US.UTF-8"
@@ -29,6 +31,9 @@ ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
 image_install_localepurge_download[weight] = "40"
 image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 image_install_localepurge_download() {
+    debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     sudo -E chroot '${ROOTFSDIR}' \
         /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only localepurge
 }
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 7c84505f..f98cd759 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -6,6 +6,7 @@
 # This file extends the image.bbclass to supply tools for futher imager functions
 
 inherit buildchroot
+inherit debrepo
 
 IMAGER_INSTALL ??= ""
 IMAGER_BUILD_DEPS ??= ""
@@ -21,14 +22,22 @@ do_install_imager_deps() {
     fi
 
     distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+    debrepo_workdir=${DEBREPO_TARGET_DIR}
     if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
         distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+        if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+            debrepo_workdir=${DEBREPO_HOST_DIR}
+        fi
     fi
 
     buildchroot_do_mounts
 
     E="${@ isar_export_proxies(d)}"
     deb_dl_dir_import ${BUILDCHROOT_DIR} ${distro}
+
+    debrepo_add_packages "${debrepo_workdir}" "${IMAGER_INSTALL}"
+    debrepo_update_apt_source_list "${BUILDCHROOT_DIR}" "base-apt"
+
     sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
         apt-get update \
             -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index cedde239..47f07129 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -2,6 +2,7 @@
 # Copyright (c) Siemens AG, 2020
 
 inherit deb-dl-dir
+inherit debrepo
 
 ROOTFS_ARCH ?= "${DISTRO_ARCH}"
 ROOTFS_DISTRO ?= "${DISTRO}"
@@ -148,6 +149,9 @@ rootfs_install_pkgs_download[weight] = "600"
 rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
 rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 rootfs_install_pkgs_download() {
+    debrepo_add_packages "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     sudo -E chroot '${ROOTFSDIR}' \
         /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
 }
-- 
2.20.1