[v9 05/13] base-apt: Predownload packages to base-apt before install

[Aliaksei Karpovich <akarpovich@ilbers.de>]

From: Uladzimir Bely <ubely@ilbers.de>

This patch uses debrepo script to predownload packages to base-apt
repository before they are installed in rootfs.

Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
Signed-off-by: Aliaksei Karpovich <akarpovich@ilbers.de>
---
 meta/classes-recipe/crossvars.bbclass         |  1 +
 meta/classes-recipe/dpkg-base.bbclass         |  1 +
 meta/classes-recipe/dpkg.bbclass              |  8 ++++
 .../image-locales-extension.bbclass           |  5 ++
 .../image-tools-extension.bbclass             | 13 ++++++
 meta/classes-recipe/rootfs.bbclass            |  5 +-
 meta/lib/aptsrc_fetcher.py                    | 46 +++++++++++++++----
 .../sbuild-chroot/sbuild-chroot-host.bb       |  2 +
 8 files changed, 72 insertions(+), 9 deletions(-)

diff --git a/meta/classes-recipe/crossvars.bbclass b/meta/classes-recipe/crossvars.bbclass
index 7cf2d660..6f070714 100644
--- a/meta/classes-recipe/crossvars.bbclass
+++ b/meta/classes-recipe/crossvars.bbclass
@@ -30,6 +30,7 @@ python __anonymous() {
         schroot_dir = d.getVar('SCHROOT_HOST_DIR', False)
         sbuild_dep = "sbuild-chroot-host" + flavor_suffix + ":do_build"
         sdk_toolchain = "crossbuild-essential-" + distro_arch
+        d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR'))
     else:
         d.setVar('BUILD_ARCH', distro_arch)
         schroot_dir = d.getVar('SCHROOT_TARGET_DIR', False)
diff --git a/meta/classes-recipe/dpkg-base.bbclass b/meta/classes-recipe/dpkg-base.bbclass
index 4a81787e..91b17768 100644
--- a/meta/classes-recipe/dpkg-base.bbclass
+++ b/meta/classes-recipe/dpkg-base.bbclass
@@ -11,6 +11,7 @@ inherit terminal
 inherit repository
 inherit deb-dl-dir
 inherit essential
+inherit debrepo
 
 # Local (WORKDIR-internal) dir where do_dpkg_build collects the built debs.
 # Its content is exported into the shared, sstate-tracked DEPLOY_DIR_DEB so
diff --git a/meta/classes-recipe/dpkg.bbclass b/meta/classes-recipe/dpkg.bbclass
index a42703d5..323202e4 100644
--- a/meta/classes-recipe/dpkg.bbclass
+++ b/meta/classes-recipe/dpkg.bbclass
@@ -117,6 +117,12 @@ dpkg_runbuild() {
     echo '$stalled_pkg_timeout = ${DPKG_BUILD_TIMEOUT};' >> ${SBUILD_CONFIG}
 
     DSC_FILE=$(find ${DEPLOY_DIR_SRC} -maxdepth 1 -name "${DEBIAN_SOURCE}_*.dsc" -print)
+    debrepo_parse_dscfile "${DSC_FILE}"
+
+    locked_update_cmd=":"
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        locked_update_cmd="flock -x /base-apt/repo.lock -c 'apt-get -y update'"
+    fi
 
     # networking is automatically enabled on older versions of sbuild
     sbuild_network_option=""
@@ -139,9 +145,11 @@ dpkg_runbuild() {
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
         --chroot-setup-commands="mkdir -p ${deb_dir}" \
         --chroot-setup-commands="find ${ext_deb_dir} -maxdepth 1 -name '*.deb' -exec ln -t ${deb_dir}/ -sf {} +" \
+        --chroot-setup-commands="${locked_update_cmd}" \
         --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-*-dummy_*.deb" \
         --finished-build-commands="find ${deb_dir} -maxdepth 1 -type f -name '*.deb' -print -exec cp ${CP_FLAGS} -t ${ext_deb_dir}/ {} +" \
+        --finished-build-commands="mkdir -p ${ext_root}" \
         ${@ '--finished-build-commands="cp /var/log/dpkg.log $ext_root/dpkg_partial.log"' if d.getVar('ISAR_CHROOT_MODE') == 'schroot' else '' } \
         --build-path="" --build-dir=${WORKDIR} --dist="${DEBDISTRONAME}" ${DSC_FILE}
 
diff --git a/meta/classes-recipe/image-locales-extension.bbclass b/meta/classes-recipe/image-locales-extension.bbclass
index c03f34c0..32b5e19c 100644
--- a/meta/classes-recipe/image-locales-extension.bbclass
+++ b/meta/classes-recipe/image-locales-extension.bbclass
@@ -6,6 +6,8 @@
 # This class extends the image.bbclass for setting locales and purging unneeded
 # ones.
 
+inherit debrepo
+
 LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
                en_US ISO-8859-1\n"
 LOCALE_DEFAULT ?= "en_US.UTF-8"
@@ -29,6 +31,9 @@ ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
 image_install_localepurge_download[weight] = "40"
 image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 image_install_localepurge_download() {
+    debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     run_privileged_heredoc <<'EOF'
     set -e
     ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), d.getVar('ROOTFS_MOUNTS') if d.getVar('ISAR_CHROOT_MODE') == 'unshare' else '')}
diff --git a/meta/classes-recipe/image-tools-extension.bbclass b/meta/classes-recipe/image-tools-extension.bbclass
index 8f666444..229ce318 100644
--- a/meta/classes-recipe/image-tools-extension.bbclass
+++ b/meta/classes-recipe/image-tools-extension.bbclass
@@ -6,6 +6,11 @@
 # This file extends the image.bbclass to supply tools for futher imager functions
 
 inherit sbuild
+inherit debrepo
+
+python __anonymous() {
+    d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR'))
+}
 
 IMAGER_INSTALL ??= ""
 IMAGER_BUILD_DEPS ??= ""
@@ -60,12 +65,20 @@ imager_run_schroot() {
         echo "Installing imager deps: ${local_install}"
 
         distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+        debrepo_workdir=${DEBREPO_TARGET_DIR}
         if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
             distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+            if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+                debrepo_workdir=${DEBREPO_HOST_DIR}
+            fi
         fi
 
         E="${@ isar_export_proxies(d)}"
         deb_dl_dir_import ${schroot_dir} ${distro}
+
+        debrepo_add_packages --isarapt "${debrepo_workdir}" "${local_install}"
+        debrepo_update_apt_source_list "${schroot_dir}" "base-apt"
+
         ${SCRIPTSDIR}/lockrun.py -r -f "${REPO_ISAR_DIR}/isar.lock" -s <<EOAPT
         schroot -r -c ${session_id} -d / -u root -- sh -c " \
             apt-get update \
diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index 99328b12..d1f24d68 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -2,7 +2,7 @@
 # Copyright (c) Siemens AG, 2020
 
 inherit deb-dl-dir
-
+inherit debrepo
 inherit sbom
 
 ROOTFS_ARCH ?= "${DISTRO_ARCH}"
@@ -344,6 +344,9 @@ rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadPro
 rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
 rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK}"
 rootfs_install_pkgs_download() {
+    debrepo_add_packages --isarapt "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     # download packages using apt in a non-privileged namespace
     rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
                ${ROOTFSDIR} \
diff --git a/meta/lib/aptsrc_fetcher.py b/meta/lib/aptsrc_fetcher.py
index fe8f1cd3..c81517c2 100644
--- a/meta/lib/aptsrc_fetcher.py
+++ b/meta/lib/aptsrc_fetcher.py
@@ -43,18 +43,29 @@ class AptSrcSchroot(AptSrc):
         repo_isar_dir = d.getVar('REPO_ISAR_DIR')
         lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
 
+        debrepo_target_dir = d.getVar('DEBREPO_TARGET_DIR')
+        isar_prefetch_base_apt = d.getVar('ISAR_PREFETCH_BASE_APT')
+        repo_base_dir = d.getVar('REPO_BASE_DIR')
+        scriptsdir = d.getVar('SCRIPTSDIR')
+
         try:
             runfetchcmd(f'''
                 set -e
+
                 schroot -r -c {session_id} -d / -u root -- \
                     rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
-                schroot -r -c {session_id} -d / -- \
-                    sh -c '
-                        set -e
-                        mkdir -p /downloads/{ud.localfile}
-                        cd /downloads/{ud.localfile}
-                        apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
-                        '
+
+                if [ "{isar_prefetch_base_apt}" = "1" ]; then
+                    {scriptsdir}/deb-repo --workdir={debrepo_target_dir} --srcmode "{ud.src_package}"
+                else
+                    schroot -r -c {session_id} -d / -- \
+                        sh -c '
+                            set -e
+                            mkdir -p /downloads/{ud.localfile}
+                            cd /downloads/{ud.localfile}
+                            apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
+                            '
+                fi
                 ''', d)
         except (OSError, FetchError):
             raise
@@ -72,9 +83,19 @@ class AptSrcSchroot(AptSrc):
 
         pp = d.getVar('PP')
         pps = d.getVar('PPS')
+
+        isar_prefetch_base_apt = d.getVar('ISAR_PREFETCH_BASE_APT')
+        repo_base_dir = d.getVar('REPO_BASE_DIR')
+
         try:
             runfetchcmd(f'''
                 set -e
+                if [ "{isar_prefetch_base_apt}" = "1" ]; then
+                    flock -x "{repo_base_dir}/repo.lock" -c "
+                    schroot -r -c {session_id} -d / -u root -- \
+                        sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+                    "
+                fi
                 schroot -r -c {session_id} -d / -u root -- \
                     rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
                 schroot -r -c {session_id} -d / -- \
@@ -82,7 +103,16 @@ class AptSrcSchroot(AptSrc):
                         set -e
                         dscfile=$(apt-get -y -qq --print-uris --only-source source {ud.src_package} | \
                                   cut -d " " -f2 | grep -E "\.dsc")
-                        cp /downloads/{ud.localfile}/* {pp}
+
+                        if [ "{isar_prefetch_base_apt}" = "1" ]; then
+                            temp_dir=$(mktemp -d)
+                            cd "$temp_dir"
+                            apt-get -y -o Debug::NoLocking=1 --download-only --only-source source {ud.src_package}
+                            # Getting rid of links
+                            cp -L "$temp_dir"/* {pp}
+                        else
+                            cp /downloads/{ud.localfile}/* {pp}
+                        fi
                         cd {pp}
                         mv -f {pps} {pps}.prev
                         dpkg-source -x "$dscfile" {pps}
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
index 52a461fe..48ce2209 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
@@ -18,3 +18,5 @@ SBUILD_CHROOT_PREINSTALL ?= " \
     ${@ 'libc6-dev:${DISTRO_ARCH} crossbuild-essential-${DISTRO_ARCH}' if bb.utils.to_boolean(d.getVar('ISAR_CROSS_COMPILE')) else ''} \
     apt-utils \
     "
+
+DEBREPO_WORKDIR = "${DEBREPO_HOST_DIR}"
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260820133934.383231-6-akarpovich%40ilbers.de.