[PATCH v7 05/10] base-apt: Predownload packages to base-apt before install

[Uladzimir Bely <ubely@ilbers.de>]

This patch uses debrepo script to predownload packages to base-apt
repository before they are installed in rootfs.

Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
 meta/classes/crossvars.bbclass                |  1 +
 meta/classes/dpkg-base.bbclass                | 27 ++++++++++++++++++-
 meta/classes/dpkg.bbclass                     |  8 ++++++
 meta/classes/image-locales-extension.bbclass  |  5 ++++
 meta/classes/image-tools-extension.bbclass    | 13 +++++++++
 meta/classes/rootfs.bbclass                   |  4 +++
 .../sbuild-chroot/sbuild-chroot-host.bb       |  2 ++
 7 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/meta/classes/crossvars.bbclass b/meta/classes/crossvars.bbclass
index 00326c9a..03a74ebe 100644
--- a/meta/classes/crossvars.bbclass
+++ b/meta/classes/crossvars.bbclass
@@ -27,6 +27,7 @@ python __anonymous() {
         schroot_dir = d.getVar('SCHROOT_HOST_DIR', False)
         sbuild_dep = "sbuild-chroot-host" + flavor_suffix + ":do_build"
         sdk_toolchain = "crossbuild-essential-" + distro_arch
+        d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR'))
     else:
         d.setVar('BUILD_ARCH', distro_arch)
         schroot_dir = d.getVar('SCHROOT_TARGET_DIR', False)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 789d6c74..367ea52d 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -11,6 +11,7 @@ inherit terminal
 inherit repository
 inherit deb-dl-dir
 inherit essential
+inherit debrepo
 
 DEPENDS ?= ""
 RPROVIDES ?= "${PROVIDES}"
@@ -114,6 +115,14 @@ do_apt_fetch() {
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap 'schroot_cleanup' EXIT
 
+    debrepo_add_packages --srcmode "${DEBREPO_TARGET_DIR}" "${SRC_APT}"
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+        schroot -r -c ${session_id} -d / -u root -- \
+            sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+        "
+    fi
+
     schroot -r -c ${session_id} -d / -u root -- \
         rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
     schroot -r -c ${session_id} -d / -- \
@@ -136,18 +145,31 @@ do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 
 # Add dependency from the correct schroot: host or target
 do_apt_fetch[depends] += "${SCHROOT_DEP}"
+# Debrepo context is created by target bootstrap, need this dependency too
+do_apt_fetch[depends] += "isar-bootstrap-target:do_bootstrap"
 
 do_apt_unpack() {
     rm -rf ${S}
     schroot_create_configs
 
+    session_id=$(schroot -b -c ${SBUILD_CHROOT})
+    echo "Started session: ${session_id}"
+
     schroot_cleanup() {
+        schroot -q -f -e -c ${session_id} > /dev/null 2>&1
         schroot_delete_configs
     }
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap 'schroot_cleanup' EXIT
 
-    schroot -d / -c ${SBUILD_CHROOT} -- \
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        flock -x "${REPO_BASE_DIR}/repo.lock" -c "
+        schroot -r -c ${session_id} -d / -u root -- \
+            sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" '
+        "
+    fi
+
+    schroot -r -c ${session_id} -d / -- \
         sh -c '
             set -e
             for uri in $2; do
@@ -157,6 +179,9 @@ do_apt_unpack() {
                 dpkg-source -x "${dscfile}" "${PPS}"
             done' \
                 my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${SRC_APT}"
+
+    # End chroot session
+    schroot -e -c ${session_id}
     schroot_delete_configs
 }
 do_apt_unpack[network] = "${TASK_USE_SUDO}"
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index bcc3f828..bf3994a6 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -111,6 +111,12 @@ dpkg_runbuild() {
     echo '$stalled_pkg_timeout = ${DPKG_BUILD_TIMEOUT};' >> ${SBUILD_CONFIG}
 
     DSC_FILE=$(find ${WORKDIR} -maxdepth 1 -name "${DEBIAN_SOURCE}_*.dsc" -print)
+    debrepo_parse_dscfile "${DSC_FILE}"
+
+    locked_update_cmd=":"
+    if [ "${ISAR_PREFETCH_BASE_APT}" = "1" ]; then
+        locked_update_cmd="flock -x /base-apt/repo.lock -c 'apt-get -y update'"
+    fi
 
     sbuild -A -n -c ${SBUILD_CHROOT} \
         --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \
@@ -122,9 +128,11 @@ dpkg_runbuild() {
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
         --chroot-setup-commands="mkdir -p ${deb_dir}" \
         --chroot-setup-commands="find ${ext_deb_dir} -maxdepth 1 -name '*.deb' -exec ln -t ${deb_dir}/ -sf {} +" \
+        --chroot-setup-commands="${locked_update_cmd}" \
         --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
         --finished-build-commands="find ${deb_dir} -maxdepth 1 -type f -name '*.deb' -print -exec cp ${CP_FLAGS} -t ${ext_deb_dir}/ {} +" \
+        --finished-build-commands="mkdir -p ${ext_root}" \
         --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
         --build-dir=${WORKDIR} --dist="isar" ${DSC_FILE}
 
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 9149d643..4dd93f1f 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -6,6 +6,8 @@
 # This class extends the image.bbclass for setting locales and purging unneeded
 # ones.
 
+inherit debrepo
+
 LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
                en_US ISO-8859-1\n"
 LOCALE_DEFAULT ?= "en_US.UTF-8"
@@ -29,6 +31,9 @@ ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
 image_install_localepurge_download[weight] = "40"
 image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 image_install_localepurge_download() {
+    debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     sudo -E chroot '${ROOTFSDIR}' \
         /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only localepurge
 }
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index e8ace8f5..ddb046a8 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -6,6 +6,11 @@
 # This file extends the image.bbclass to supply tools for futher imager functions
 
 inherit sbuild
+inherit debrepo
+
+python __anonymous() {
+    d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_TARGET_DIR'))
+}
 
 IMAGER_INSTALL ??= ""
 IMAGER_BUILD_DEPS ??= ""
@@ -40,12 +45,20 @@ imager_run() {
         echo "Installing imager deps: ${local_install}"
 
         distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+        debrepo_workdir=${DEBREPO_TARGET_DIR}
         if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
             distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+            if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then
+                debrepo_workdir=${DEBREPO_HOST_DIR}
+            fi
         fi
 
         E="${@ isar_export_proxies(d)}"
         deb_dl_dir_import ${schroot_dir} ${distro}
+
+        debrepo_add_packages --isarapt "${debrepo_workdir}" "${local_install}"
+        debrepo_update_apt_source_list "${schroot_dir}" "base-apt"
+
         ${SCRIPTSDIR}/lockrun.py -r -f "${REPO_ISAR_DIR}/isar.lock" -s <<EOAPT
         schroot -r -c ${session_id} -d / -u root -- sh -c " \
             apt-get update \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 7d0bc0c8..f41fb498 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -2,6 +2,7 @@
 # Copyright (c) Siemens AG, 2020
 
 inherit deb-dl-dir
+inherit debrepo
 
 ROOTFS_ARCH ?= "${DISTRO_ARCH}"
 ROOTFS_DISTRO ?= "${DISTRO}"
@@ -153,6 +154,9 @@ rootfs_install_pkgs_download[weight] = "600"
 rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
 rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 rootfs_install_pkgs_download() {
+    debrepo_add_packages --isarapt "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}"
+    debrepo_update_apt_source_list "${ROOTFSDIR}" "base-apt"
+
     sudo -E chroot '${ROOTFSDIR}' \
         /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
 }
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
index 255d6937..6734eb18 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot-host.bb
@@ -24,3 +24,5 @@ SBUILD_CHROOT_PREINSTALL:riscv64 ?= " \
     gcc-riscv64-linux-gnu \
     g++-riscv64-linux-gnu \
     dpkg-cross"
+
+DEBREPO_WORKDIR = "${DEBREPO_HOST_DIR}"
-- 
2.44.2

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/isar-users/20240725151006.2129-6-ubely%40ilbers.de.