[PATCHv5 23/29] base-apt: pull base-apt population to the front of the build chain

[Henning Schild <henning.schild@siemens.com>]

From: Henning Schild <henning.schild@siemens.com>

Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.

The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.

Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
 doc/user_manual.md                            |  8 +--
 meta/classes/image-cache-extension.bbclass    | 64 ------------------
 meta/classes/image.bbclass                    |  1 -
 meta/classes/rootfs.bbclass                   |  2 +-
 meta/conf/bitbake.conf                        |  4 +-
 .../isar-bootstrap/isar-bootstrap.inc         |  1 +
 meta/recipes-devtools/base-apt/base-apt.bb    | 67 ++++++++++++++++---
 scripts/ci_build.sh                           |  4 +-
 8 files changed, 69 insertions(+), 82 deletions(-)
 delete mode 100644 meta/classes/image-cache-extension.bbclass

diff --git a/doc/user_manual.md b/doc/user_manual.md
index 8ba92eed..d13a74e9 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -836,10 +836,10 @@ path to the public key in `conf/local.conf`, e.g.:
 BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
 ```
 
- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.
 
 ```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
 ```
 
  - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -849,14 +849,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
 #ISAR_USE_CACHED_BASE_REPO ?= "1"
 #BB_NO_NETWORK ?= "1"
 ```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.
 
 ```
 sudo rm -rf tmp
 
 ```
 
- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):
 
 ```
 bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326e..00000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
-    find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
-        # NOTE: due to packages stored by reprepro are not modified, we can
-        # use search by filename to check if package is already in repo. In
-        # addition, md5sums are compared to ensure that the package is the
-        # same and should not be overwritten. This method is easier and more
-        # robust than querying reprepro by name.
-
-        # Check if this package is taken from Isar-apt, if so - ingore it.
-        repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
-            continue
-
-        # Check if this package is already in base-apt
-        ret=0
-	repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" || 
-	    ret=$?
-        [ "${ret}" = "0" ] && continue
-        if [ "${ret}" = "1" ]; then
-            repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
-                "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
-                "${BASE_DISTRO_CODENAME}" \
-                "${base_apt_p}"
-        fi
-
-        repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
-            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
-            "${BASE_DISTRO_CODENAME}" \
-            "${package}"
-    done
-
-    find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
-        repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
-            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
-            "${BASE_DISTRO_CODENAME}" \
-            "${package}"
-    done
-}
-
-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
-    if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
-        [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
-        # We would need two separate repository paths for that.
-        # Otherwise packages (especially the 'all' arch ones) from one
-        # distribution can influence the package versions of the other
-        # distribution.
-        bbfatal "Different host and target distributions are currently not supported." \
-                "Try it without cross-build."
-    fi
-
-    populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index adaca5db..96ba863e 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 
 inherit rootfs
 inherit image-sdk-extension
-inherit image-cache-extension
 inherit image-tools-extension
 inherit image-postproc-extension
 inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index be101890..806e824c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
 
 do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
 do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
 do_rootfs_install[deptask] = "do_deploy_deb"
 python do_rootfs_install() {
     configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index b7c0a8f5..848e085a 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -82,8 +82,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
 THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
 
 # Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
 
 BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
     SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2078115b..a93c4e3b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():
 
 do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
 do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] = "base-apt:do_cache"
 
 isar_bootstrap() {
     deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 7c891fa0..16fc5f22 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
 BASE_REPO_KEY ?= ""
 KEYFILES ?= ""
 
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+    find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+        # NOTE: due to packages stored by reprepro are not modified, we can
+        # use search by filename to check if package is already in repo. In
+        # addition, md5sums are compared to ensure that the package is the
+        # same and should not be overwritten. This method is easier and more
+        # robust than querying reprepro by name.
 
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+        # Check if this package is taken from Isar-apt, if so - ingore it.
+        repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+            continue
+
+        # Check if this package is already in base-apt
+        ret=0
+	repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+	    ret=$?
+        [ "${ret}" = "0" ] && continue
+        if [ "${ret}" = "1" ]; then
+            repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+                "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+                "${BASE_DISTRO_CODENAME}" \
+                "${base_apt_p}"
+        fi
+
+        repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+            "${BASE_DISTRO_CODENAME}" \
+            "${package}"
+    done
+
+    find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+        repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+            "${BASE_DISTRO_CODENAME}" \
+            "${package}"
+    done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
     repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
         "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
         "${BASE_DISTRO_CODENAME}" \
         "${KEYFILES}"
+
+    if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+        [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+        # We would need two separate repository paths for that.
+        # Otherwise packages (especially the 'all' arch ones) from one
+        # distribution can influence the package versions of the other
+        # distribution.
+        bbfatal "Different host and target distributions are currently not supported." \
+                "Try it without cross-build."
+    fi
+
+    populate_base_apt
 }
 
-python do_cache_config() {
+python do_cache() {
+    if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+        return 0
+
     for key in d.getVar('BASE_REPO_KEY').split():
         d.appendVar("SRC_URI", " %s" % key)
         fetcher = bb.fetch2.Fetch([key], d)
         filename = fetcher.localpath(key)
         d.appendVar("KEYFILES", " %s" % filename)
 
-    bb.build.exec_func('repo_config', d)
+    bb.build.exec_func('repo', d)
 }
 
-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee91..733ba86e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
 
     # Enable use of signed cached base repository
     echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
-    bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+    bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
     while [ -e bitbake.sock ]; do sleep 1; done
     sudo rm -rf tmp
     sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
     sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
 
     # Enable use of unsigned cached base repository
-    bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+    bitbake $BB_ARGS $REPRO_TARGETS_SET
     while [ -e bitbake.sock ]; do sleep 1; done
     sudo rm -rf tmp
     sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
-- 
2.24.1