[PATCH 2/8] rootfs: fold do_rootfs_postprocess task into do_rootfs_install

["'Felix Moessbauer' via isar-users" <isar-users@googlegroups.com>]

The do_rootfs_postprocess task is not idempotent and contains steps
which can only run on an unmodified rootfs from do_rootfs_install (e.g.
user-creation, steps that need the apt cache). To make both steps
atomic, we move them into the same task, similar to how OE models
it.

This further improves build performance on rebuilds from the sstate
cache, as the whole artifact can be taken and the postprocess steps do
not need to be re-executed. The sstate artifact is also smaller, as the
rootfs is stored in clean state.

By that, we further drop the manual registration of the USERS / GROUPS
vardeps on the do_rootfs_install task. These are now propagated via the
corresponding ROOTFS_POSTPROCESS_COMMAND.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 RECIPE-API-CHANGELOG.md                       |  6 +++
 .../image-account-extension.bbclass           |  3 --
 meta/classes-recipe/image.bbclass             |  4 +-
 meta/classes-recipe/initramfs.bbclass         |  2 +-
 meta/classes-recipe/rootfs.bbclass            | 42 +++++--------------
 5 files changed, 20 insertions(+), 37 deletions(-)

diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index 2db05169..f5630762 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -1201,3 +1201,9 @@ To opt out, remove the feature explicitly:
 ```
 ROOTFS_FEATURES:remove = "clean-apt-credentials"
 ```
+
+### do_rootfs_postprocess is folded into do_rootfs_install
+
+All `ROOTFS_POSTPROCESS_COMMAND` steps are now executed within the `do_rootfs_install`
+task to avoid idempotency issues on partial rebuilds. Task that previously had a
+dependency to `do_rootfs_postprocess` shall now be changed to run after `do_rootfs_install`.
diff --git a/meta/classes-recipe/image-account-extension.bbclass b/meta/classes-recipe/image-account-extension.bbclass
index de01484c..dd70f2a3 100644
--- a/meta/classes-recipe/image-account-extension.bbclass
+++ b/meta/classes-recipe/image-account-extension.bbclass
@@ -14,14 +14,11 @@ python() {
     for entry in (d.getVar("GROUPS") or "").split():
         group_entry = "GROUP_{}".format(entry)
         d.appendVarFlag("image_postprocess_accounts", "vardeps", " {}".format(group_entry))
-        d.appendVarFlag("do_rootfs_install", "vardeps", " {}".format(group_entry))
 
     for entry in (d.getVar("USERS") or "").split():
         user_entry = "USER_{}".format(entry)
         d.appendVarFlag("image_postprocess_accounts", "vardeps", " {}".format(user_entry))
-        d.appendVarFlag("do_rootfs_install", "vardeps", " {}".format(user_entry))
 }
-do_rootfs_install[vardeps] += "GROUPS USERS"
 
 def image_create_groups(d: "DataSmart") -> None:
     """Creates the groups defined in the ``GROUPS`` bitbake variable.
diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass
index d02f99da..a749441f 100644
--- a/meta/classes-recipe/image.bbclass
+++ b/meta/classes-recipe/image.bbclass
@@ -408,7 +408,7 @@ do_copy_boot_files() {
         cp -f "$dtb" "${DEPLOYDIR}/"
     done
 }
-addtask copy_boot_files before do_rootfs_postprocess after do_rootfs_install
+addtask copy_boot_files before do_rootfs do_generate_initramfs after do_rootfs_install
 
 python do_copy_boot_files_setscene () {
     sstate_setscene(d)
@@ -492,7 +492,7 @@ EOSUDO
         -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';'
 }
 do_rootfs_finalize[network] = "${TASK_USE_SUDO}"
-addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
+addtask rootfs_finalize before do_rootfs after do_rootfs_install
 
 ROOTFS_QA_FIND_ARGS ?= ""
 
diff --git a/meta/classes-recipe/initramfs.bbclass b/meta/classes-recipe/initramfs.bbclass
index 5a98e337..a647b6ea 100644
--- a/meta/classes-recipe/initramfs.bbclass
+++ b/meta/classes-recipe/initramfs.bbclass
@@ -78,7 +78,7 @@ rootfs_purge_initramfs() {
 
 # If an external initrd shall be used, run it as a task
 # instead of as part of the rootfs install.
-addtask do_generate_initramfs before do_rootfs_postprocess after do_rootfs_install
+addtask do_generate_initramfs before do_rootfs after do_rootfs_install
 addtask do_generate_initramfs_setscene
 ROOTFS_INSTALL_COMMAND:remove = "rootfs_generate_initramfs"
 
diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index 8d394b05..b85cb59a 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -430,19 +430,21 @@ do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
 do_rootfs_install[cleandirs] += "${DEPLOYDIR}"
 do_rootfs_install[sstate-inputdirs] = "${DEPLOYDIR}"
 do_rootfs_install[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
-do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND} ${ROOTFS_VARDEPS}"
+do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND} ${ROOTFS_POSTPROCESS_COMMAND} ${ROOTFS_VARDEPS}"
 do_rootfs_install[vardepsexclude] += "IMAGE_ROOTFS"
 do_rootfs_install[depends] = "bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
+do_rootfs_install[depends] += "base-apt:do_cache isar-apt:do_cache_config"
 do_rootfs_install[deptask] = "do_deploy_deb"
 do_rootfs_install[rdeptask] = "do_deploy_deb"
 do_rootfs_install[network] = "${TASK_USE_SUDO}"
 python do_rootfs_install() {
     configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND") or "").split()
     install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND") or "").split()
+    postprocess_cmds = (d.getVar("ROOTFS_POSTPROCESS_COMMAND") or "").split()
 
     # Mount after configure commands, so that they have time to copy
     # 'isar-apt' (sdkchroot):
-    cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
+    cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds + postprocess_cmds
 
     # NOTE: The weights specify how long each task takes in seconds and are used
     # by the MultiStageProgressReporter to render a progress bar for this task.
@@ -470,7 +472,7 @@ python do_rootfs_install() {
         progress_reporter.finish()
         bb.build.exec_func('rootfs_do_umounts', d)
 }
-addtask rootfs_install before do_rootfs_postprocess after do_unpack
+addtask rootfs_install before do_rootfs after do_unpack
 
 do_cache_deb_src[network] = "${TASK_USE_SUDO}"
 do_cache_deb_src() {
@@ -611,33 +613,6 @@ image_postprocess_populate_systemd_preset() {
 EOSH
 }
 
-do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
-do_rootfs_postprocess[network] = "${TASK_USE_SUDO}"
-do_rootfs_postprocess[depends] = "base-apt:do_cache isar-apt:do_cache_config"
-python do_rootfs_postprocess() {
-    # Take care that its correctly mounted:
-    bb.build.exec_func('rootfs_do_mounts', d)
-    # Take care that qemu-*-static is available, since it could have been
-    # removed on a previous execution of this task:
-    bb.build.exec_func('rootfs_do_qemu', d)
-
-    progress_reporter = bb.progress.ProgressHandler(d)
-    progress_reporter.update(0)
-
-    cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
-    if cmds is None or not cmds.strip():
-        return
-    cmds = cmds.split()
-
-    try:
-        for i, cmd in enumerate(cmds):
-            bb.build.exec_func(cmd, d)
-            progress_reporter.update(int(i / len(cmds) * 100))
-    finally:
-        bb.build.exec_func('rootfs_do_umounts', d)
-}
-addtask rootfs_postprocess before do_rootfs after do_unpack
-
 ROOTFS_INSTALL_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-initrd', 'rootfs_generate_initramfs', '', d)}"
 rootfs_generate_initramfs[weight] = "1000"
 rootfs_generate_initramfs[progress] = "custom:rootfs_progress.InitrdProgressHandler"
@@ -682,7 +657,7 @@ rootfs_install_sstate_prepare() {
 
     run_privileged_heredoc <<'EOF'
         mount -o bind,private '${ROOTFSDIR}' '${WORKDIR}/mnt/rootfs' -o ro
-        lopts="--one-file-system --exclude=var/cache/apt/archives"
+        lopts="--one-file-system"
         tar -C ${WORKDIR}/mnt/rootfs -cpSf rootfs.tar $lopts ${SSTATE_TAR_ATTR_FLAGS} .
         umount -q ${WORKDIR}/mnt/rootfs
 EOF
@@ -715,3 +690,8 @@ python do_rootfs_install_setscene() {
     sstate_setscene(d)
 }
 addtask do_rootfs_install_setscene
+
+python do_rootfs_postprocess() {
+    bb.warn("task do_rootfs_postprocess was folded into do_rootfs_install, please order your task after do_rootfs_install instead")
+}
+addtask do_rootfs_postprocess after do_rootfs_install
-- 
2.55.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260820083014.3377283-3-felix.moessbauer%40siemens.com.