[PATCH 0/6] Rework SBOM generation

[PATCH 0/6] Rework SBOM generation

['Felix Moessbauer' via isar-users]

The SBOM generation currently fails to capture the package checksums
in case of partial rebuilds. This is because the apt-state is not
captured before the rootfs cleanup.

We fix this by capturing the apt-state (and copyright data) while it
still is available. Then, we run the SBOM generator against that data
in a dedicated task. By that, the SBOM generation also can be cached,
if the rootfs does not change - speeding up the build on repeated runs.

We further fix a SBOM chroot deploy race on multi-config builds.

Best regards,
Felix Moessbauer

Felix Moessbauer (6):
  rootfs: capture apt-state before rootfs cleanup
  sbom: run generator as task with apt-cache as input
  wic: make dependency to sbom chroot explicit
  sbom: deploy sbom chroot to distro specific file
  sbom: align deploy dir names
  sbom: cache artifact in sstate cache

 meta/classes-recipe/imagetypes_wic.bbclass |  1 +
 meta/classes-recipe/rootfs.bbclass         | 27 +++++++++++++---
 meta/classes/sbom.bbclass                  | 37 ++++++++++++++++------
 3 files changed, 51 insertions(+), 14 deletions(-)

-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-1-felix.moessbauer%40siemens.com.

[PATCH 1/6] rootfs: capture apt-state before rootfs cleanup

['Felix Moessbauer' via isar-users]

The do_rootfs_postprocess command removes parts of the apt cache which
we need for SBOM generation. On normal builds this is not a problem, as
the SBOM generator runs prior to the rootfs cleanup. However, on partial
rebuilds this can result in incomplete SBOMs which cannot be easily
detected.

This further allows to add more parallelism to the build, as the SBOM
generator can run anytime after do_rootfs_install (not part of this
patch).

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes-recipe/rootfs.bbclass | 21 +++++++++++++++++++++
 meta/classes/sbom.bbclass          | 10 ++++++----
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index e17f711f..16b1aa50 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -42,6 +42,8 @@ ROOTFS_FEATURES:remove:bullseye = "generate-sbom"
 ROOTFS_FEATURES:remove:jammy = "generate-sbom"
 ROOTFS_FEATURES:remove:focal = "generate-sbom"
 
+# Capture all information needed for sbom generation
+ROOTFS_APT_STATE = "apt-state.tar.zst"
 ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
 
 ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
@@ -414,6 +416,19 @@ rootfs_clear_initrd_symlinks() {
     run_privileged rm -f ${ROOTFSDIR}/initrd.img.old
 }
 
+ROOTFS_INSTALL_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-sbom', 'rootfs_capture_apt_state', '', d)}"
+rootfs_capture_apt_state() {
+    ( cd ${ROOTFSDIR} && find usr/share/doc -name copyright -print0 ) | \
+    tar -cf ${WORKDIR}/${ROOTFS_APT_STATE} --zstd -C ${ROOTFSDIR} \
+        --exclude=var/lib/apt/lists/partial \
+        --exclude=var/lib/apt/lists/lock \
+        --exclude=var/lib/apt/lists/auxfiles \
+        --null -T - \
+        var/lib/apt/lists \
+        var/lib/apt/extended_states \
+        var/lib/dpkg/status
+}
+
 do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
 do_rootfs_install[cleandirs] += "${DEPLOYDIR}"
 do_rootfs_install[sstate-inputdirs] = "${DEPLOYDIR}"
@@ -674,6 +689,9 @@ rootfs_install_sstate_prepare() {
         tar -C ${WORKDIR}/mnt/rootfs -cpSf rootfs.tar $lopts ${SSTATE_TAR_ATTR_FLAGS} .
         umount -q ${WORKDIR}/mnt/rootfs
 EOF
+    if [ -f ${WORKDIR}/${ROOTFS_APT_STATE} ]; then
+        cp ${WORKDIR}/${ROOTFS_APT_STATE} .
+    fi
     ${@ 'sudo chown $(id -u):$(id -g) rootfs.tar' if d.getVar('ISAR_CHROOT_MODE') == 'schroot' else '' }
 }
 rootfs_install_sstate_prepare[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
@@ -691,6 +709,9 @@ rootfs_install_sstate_finalize() {
 EOF
         rm rootfs.tar
     fi
+    if [ -f ${ROOTFS_APT_STATE} ]; then
+        mv ${ROOTFS_APT_STATE} ${WORKDIR}/${ROOTFS_APT_STATE}
+    fi
 }
 
 python do_rootfs_install_setscene() {
diff --git a/meta/classes/sbom.bbclass b/meta/classes/sbom.bbclass
index 2e6d579f..34efb590 100644
--- a/meta/classes/sbom.bbclass
+++ b/meta/classes/sbom.bbclass
@@ -50,16 +50,18 @@ EOF
 }
 
 generate_sbom() {
-    run_privileged mkdir -p \
-        ${SBOM_CHROOT_LOCAL}/mnt/rootfs \
-        ${SBOM_CHROOT_LOCAL}/mnt/deploy-dir
+    run_privileged_heredoc <<'EOF'
+        mkdir -p ${SBOM_CHROOT_LOCAL}/mnt/rootfs \
+                 ${SBOM_CHROOT_LOCAL}/mnt/deploy-dir
+        tar -xf ${WORKDIR}/${ROOTFS_APT_STATE} --zstd \
+            -C ${SBOM_CHROOT_LOCAL}/mnt/rootfs
+EOF
 
     TIMESTAMP=$(date --iso-8601=s -d @${SOURCE_DATE_EPOCH})
     bwrap \
         --unshare-user \
         --unshare-pid \
         --bind ${SBOM_CHROOT_LOCAL} / \
-        --bind ${ROOTFSDIR} /mnt/rootfs \
         --bind ${DEPLOY_DIR_SBOM} /mnt/deploy-dir \
         -- debsbom -v generate ${SBOM_DEBSBOM_TYPE_ARGS} -r /mnt/rootfs -o /mnt/deploy-dir/'${ROOTFS_PACKAGE_SUFFIX}' \
             --distro-name '${SBOM_DISTRO_NAME}' --distro-supplier '${SBOM_DISTRO_SUPPLIER}' \
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-2-felix.moessbauer%40siemens.com.

[PATCH 2/6] sbom: run generator as task with apt-cache as input

['Felix Moessbauer' via isar-users]

As the rootfs now deploys the apt cache prior to any modification, we
can safely run the sbom generator as a standalone task. By that, we get
more parallelism into the build and avoid the inter-chroot dependency
between the sbom chroot and the image chroot.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes-recipe/rootfs.bbclass | 6 +-----
 meta/classes/sbom.bbclass          | 8 ++++++++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index 16b1aa50..a96a392a 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -519,9 +519,6 @@ cache_dbg_pkgs() {
     fi
 }
 
-# The sbom generator needs the apt-cache, hence run before cleaning it
-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-sbom', 'do_generate_sbom', '', d)}"
-
 ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
 rootfs_postprocess_clean_package_cache() {
     run_in_chroot '${ROOTFSDIR}' \
@@ -615,6 +612,7 @@ EOSH
 
 do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
 do_rootfs_postprocess[network] = "${TASK_USE_SUDO}"
+do_rootfs_postprocess[depends] = "base-apt:do_cache isar-apt:do_cache_config"
 python do_rootfs_postprocess() {
     # Take care that its correctly mounted:
     bb.build.exec_func('rootfs_do_mounts', d)
@@ -665,8 +663,6 @@ python do_rootfs() {
 }
 addtask rootfs before do_build
 
-do_rootfs_postprocess[depends] = "base-apt:do_cache isar-apt:do_cache_config ${@bb.utils.contains('ROOTFS_FEATURES', 'generate-sbom', 'sbom-chroot:do_sbomchroot_deploy', '', d)}"
-
 SSTATETASKS += "do_rootfs_install"
 SSTATECREATEFUNCS += "rootfs_install_sstate_prepare"
 SSTATEPOSTINSTFUNCS += "rootfs_install_sstate_finalize"
diff --git a/meta/classes/sbom.bbclass b/meta/classes/sbom.bbclass
index 34efb590..1801afac 100644
--- a/meta/classes/sbom.bbclass
+++ b/meta/classes/sbom.bbclass
@@ -78,6 +78,7 @@ cleanup_sbom_chroot() {
 
 do_generate_sbom[dirs] += "${DEPLOY_DIR_SBOM}"
 do_generate_sbom[network] = "${TASK_USE_SUDO}"
+do_generate_sbom[depends] += "sbom-chroot:do_sbomchroot_deploy"
 python do_generate_sbom() {
     sbom_doc_uuid(d)
     try:
@@ -86,3 +87,10 @@ python do_generate_sbom() {
     finally:
         bb.build.exec_func("cleanup_sbom_chroot", d)
 }
+
+# The sbom generator uses the apt state captured during do_rootfs_install,
+# so it can run as a standalone task afterwards
+python() {
+    if 'generate-sbom' in d.getVar('ROOTFS_FEATURES').split():
+        bb.build.addtask('do_generate_sbom', 'do_rootfs', 'do_rootfs_install', d)
+}
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-3-felix.moessbauer%40siemens.com.

[PATCH 3/6] wic: make dependency to sbom chroot explicit

['Felix Moessbauer' via isar-users]

If the rootfs feature generate-sbom is enabled, the wic task also
generates an imager sbom and merges it with the rootfs SBOM. For that,
the SBOM chroot needs to be deployed, which currently implicitly happens
as the wic task requires the do_rootfs_install task to be executed,
which in turn depends on the SBOM chroot.

We now make this dependency explicit to be able to later cache the
expensive rootfs SBOM task.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes-recipe/imagetypes_wic.bbclass | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/classes-recipe/imagetypes_wic.bbclass b/meta/classes-recipe/imagetypes_wic.bbclass
index 3c65ed0d..c1647081 100644
--- a/meta/classes-recipe/imagetypes_wic.bbclass
+++ b/meta/classes-recipe/imagetypes_wic.bbclass
@@ -143,6 +143,7 @@ check_for_wic_warnings() {
     fi
 }
 
+do_image_wic[depends] += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-sbom', 'sbom-chroot:do_sbomchroot_deploy', '', d)}"
 do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}"
 IMAGE_CMD:wic() {
     generate_wic_image
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-4-felix.moessbauer%40siemens.com.

[PATCH 4/6] sbom: deploy sbom chroot to distro specific file

['Felix Moessbauer' via isar-users]

When running under multiconfig with targets that do not share the
distribution, the sbom chroot is built multiple times and deployed to
the same file (data race).

As this does not matter much for the SBOM generation itself, this was
not detected. However, when building with sstate cache it can happen
that multiple sbom chroots are deployed roughly at the same time (racy),
breaking the build.

We fix it by making the sbom chroot rootfs file distro and architecture
specific.

Fixes: 812b6c77 ("meta: add SBOM generation with debsbom")
Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes/sbom.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/sbom.bbclass b/meta/classes/sbom.bbclass
index 1801afac..29cae780 100644
--- a/meta/classes/sbom.bbclass
+++ b/meta/classes/sbom.bbclass
@@ -23,7 +23,7 @@ SBOM_SPDX_NAMESPACE_PREFIX ?= "https://spdx.org/spdxdocs"
 DEPLOY_DIR_SBOM = "${DEPLOY_DIR_IMAGE}"
 
 SBOM_DIR = "${DEPLOY_DIR}/sbom"
-SBOM_CHROOT = "${SBOM_DIR}/sbom-chroot.tar.zst"
+SBOM_CHROOT = "${SBOM_DIR}/${HOST_DISTRO}-${HOST_ARCH}_${DISTRO}-${DISTRO_ARCH}.tar.zst"
 SBOM_CHROOT_LOCAL = "${WORKDIR}/sbom-chroot"
 
 # adapted from the isar-cip-core image_uuid.bbclass
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-5-felix.moessbauer%40siemens.com.

[PATCH 5/6] sbom: align deploy dir names

['Felix Moessbauer' via isar-users]

All chroots are deployed with either schroot- or chroot- prefix. We
now align the sbom class to this pattern.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes/sbom.bbclass | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/sbom.bbclass b/meta/classes/sbom.bbclass
index 29cae780..4f7f2251 100644
--- a/meta/classes/sbom.bbclass
+++ b/meta/classes/sbom.bbclass
@@ -22,9 +22,9 @@ SBOM_SPDX_NAMESPACE_PREFIX ?= "https://spdx.org/spdxdocs"
 
 DEPLOY_DIR_SBOM = "${DEPLOY_DIR_IMAGE}"
 
-SBOM_DIR = "${DEPLOY_DIR}/sbom"
+SBOM_DIR = "${DEPLOY_DIR}/chroot-sbom"
 SBOM_CHROOT = "${SBOM_DIR}/${HOST_DISTRO}-${HOST_ARCH}_${DISTRO}-${DISTRO_ARCH}.tar.zst"
-SBOM_CHROOT_LOCAL = "${WORKDIR}/sbom-chroot"
+SBOM_CHROOT_LOCAL = "${WORKDIR}/chroot-sbom"
 
 # adapted from the isar-cip-core image_uuid.bbclass
 def generate_document_uuid(d, warn_not_repr=True):
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-6-felix.moessbauer%40siemens.com.

[PATCH 6/6] sbom: cache artifact in sstate cache

['Felix Moessbauer' via isar-users]

By that, repeated builds of unchanged rootfs do not need to run through
the potentially expensive SBOM generation.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes/sbom.bbclass | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/meta/classes/sbom.bbclass b/meta/classes/sbom.bbclass
index 4f7f2251..b2e8aff6 100644
--- a/meta/classes/sbom.bbclass
+++ b/meta/classes/sbom.bbclass
@@ -21,6 +21,7 @@ SBOM_DEBSBOM_EXTRA_ARGS ?= "--with-licenses"
 SBOM_SPDX_NAMESPACE_PREFIX ?= "https://spdx.org/spdxdocs"
 
 DEPLOY_DIR_SBOM = "${DEPLOY_DIR_IMAGE}"
+SBOM_LOCAL_DEPLOYDIR = "${WORKDIR}/chroot-sbom-deploy"
 
 SBOM_DIR = "${DEPLOY_DIR}/chroot-sbom"
 SBOM_CHROOT = "${SBOM_DIR}/${HOST_DISTRO}-${HOST_ARCH}_${DISTRO}-${DISTRO_ARCH}.tar.zst"
@@ -62,7 +63,7 @@ EOF
         --unshare-user \
         --unshare-pid \
         --bind ${SBOM_CHROOT_LOCAL} / \
-        --bind ${DEPLOY_DIR_SBOM} /mnt/deploy-dir \
+        --bind ${SBOM_LOCAL_DEPLOYDIR} /mnt/deploy-dir \
         -- debsbom -v generate ${SBOM_DEBSBOM_TYPE_ARGS} -r /mnt/rootfs -o /mnt/deploy-dir/'${ROOTFS_PACKAGE_SUFFIX}' \
             --distro-name '${SBOM_DISTRO_NAME}' --distro-supplier '${SBOM_DISTRO_SUPPLIER}' \
             --distro-version '${SBOM_DISTRO_VERSION}' --distro-arch '${DISTRO_ARCH}' \
@@ -76,7 +77,10 @@ cleanup_sbom_chroot() {
     run_privileged rm -rf ${SBOM_CHROOT_LOCAL}
 }
 
-do_generate_sbom[dirs] += "${DEPLOY_DIR_SBOM}"
+SSTATETASKS += "do_generate_sbom"
+do_generate_sbom[cleandirs] += "${SBOM_LOCAL_DEPLOYDIR}"
+do_generate_sbom[sstate-inputdirs] = "${SBOM_LOCAL_DEPLOYDIR}"
+do_generate_sbom[sstate-outputdirs] = "${DEPLOY_DIR_SBOM}"
 do_generate_sbom[network] = "${TASK_USE_SUDO}"
 do_generate_sbom[depends] += "sbom-chroot:do_sbomchroot_deploy"
 python do_generate_sbom() {
@@ -88,9 +92,14 @@ python do_generate_sbom() {
         bb.build.exec_func("cleanup_sbom_chroot", d)
 }
 
+python do_generate_sbom_setscene() {
+    sstate_setscene(d)
+}
+
 # The sbom generator uses the apt state captured during do_rootfs_install,
 # so it can run as a standalone task afterwards
 python() {
     if 'generate-sbom' in d.getVar('ROOTFS_FEATURES').split():
         bb.build.addtask('do_generate_sbom', 'do_rootfs', 'do_rootfs_install', d)
+        bb.build.addtask('do_generate_sbom_setscene', None, None, d)
 }
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720081347.3835974-7-felix.moessbauer%40siemens.com.

Re: [PATCH 0/6] Rework SBOM generation

[Zhihang Wei]

Applied to next, thanks.

Zhihang

On 7/20/26 10:13, 'Felix Moessbauer' via isar-users wrote:
> The SBOM generation currently fails to capture the package checksums
> in case of partial rebuilds. This is because the apt-state is not
> captured before the rootfs cleanup.
>
> We fix this by capturing the apt-state (and copyright data) while it
> still is available. Then, we run the SBOM generator against that data
> in a dedicated task. By that, the SBOM generation also can be cached,
> if the rootfs does not change - speeding up the build on repeated runs.
>
> We further fix a SBOM chroot deploy race on multi-config builds.
>
> Best regards,
> Felix Moessbauer
>
> Felix Moessbauer (6):
>    rootfs: capture apt-state before rootfs cleanup
>    sbom: run generator as task with apt-cache as input
>    wic: make dependency to sbom chroot explicit
>    sbom: deploy sbom chroot to distro specific file
>    sbom: align deploy dir names
>    sbom: cache artifact in sstate cache
>
>   meta/classes-recipe/imagetypes_wic.bbclass |  1 +
>   meta/classes-recipe/rootfs.bbclass         | 27 +++++++++++++---
>   meta/classes/sbom.bbclass                  | 37 ++++++++++++++++------
>   3 files changed, 51 insertions(+), 14 deletions(-)
>

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/54cb22e2-d300-4dd4-92e8-551652b849f4%40ilbers.de.