[PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

[PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

Without this option, mmdebstrap deletes the var/log/dpkg.log and its siblings by default,
but this is needed to download deb-src packages for base-apt creation.
Basically, we are referring to the package's status under /var/log/dpkg.log file
and performing the debsrc_download operation.

Without these changes, bootstrap related deb-src files are not downloaded
under the /downloads/deb-src folder, due to this the repo sanity check
functions failed during base-apt caching stage.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 931f6f13..7380a658 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -168,6 +168,11 @@ do_bootstrap() {
         arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
     fi
 
+    cleanup=
+    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
+        cleanup="--skip=cleanup/reproducible"
+    fi
+
     # Cleanup mounts if fails
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
@@ -183,6 +188,7 @@ do_bootstrap() {
 
     sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
                    $arch_param \
+                   $cleanup \
                    --mode=unshare \
                    ${MMHOOKS} \
                    --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250305142513.1841415-1-srinuvasan.a%40siemens.com.

[PATCH 2/3] image: download deb-src packages after imager creation

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

With the present implementation, deb-src packages are missing for imager related
deb packages (packages listed in IMAGER_INSTALL).
Copy the /var/log/dpkg.log file from the sbuild environment to outside
With that, use the list to download deb-src packages for IMAGER_INSTALL
related deb packages.

Presently, cache_deb_src function running as part of ROOTFS_POSTPROCESS_COMMAND,
this will not be helpful to get the imager related deb-src packages, hence
make cache_deb_src as a separate task and running
always after do_image to get the imager deps related deb-src packages.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/classes/image-tools-extension.bbclass | 8 +++++++-
 meta/classes/image.bbclass                 | 8 +++++++-
 meta/classes/rootfs.bbclass                | 3 ++-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index e8ace8f5..156dfcb3 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -7,6 +7,9 @@
 
 inherit sbuild
 
+PP = "/home/builder/${PN}-${MACHINE}"
+ext_root="${PP}/rootfs"
+
 IMAGER_INSTALL ??= ""
 IMAGER_BUILD_DEPS ??= ""
 DEPENDS += "${IMAGER_BUILD_DEPS}"
@@ -59,9 +62,11 @@ EOAPT
 
         deb_dl_dir_export ${schroot_dir} ${distro}
         schroot -r -c ${session_id} -d / -u root -- sh -c " \
+            rm -f /var/log/dpkg.log
             apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
                 --allow-unauthenticated --allow-downgrades install \
-                ${local_install}"
+                ${local_install}
+            cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log"
     fi
 
     schroot -r -c ${session_id} "$@"
@@ -70,4 +75,5 @@ EOAPT
 
     remove_mounts
     schroot_delete_configs
+    sbuild_dpkg_log_export "${WORKDIR}/rootfs/dpkg_partial.log"
 }
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 56eca202..66b4d313 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -72,7 +72,13 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_PACKAGE_SUFFIX ?= "${PN}-${DISTRO}-${MACHINE}"
 
-ROOTFS_POSTPROCESS_COMMAND:prepend = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'cache_deb_src', '', d)} "
+CACHE_DEB_SRC = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', '1', '0', d)}"
+
+python () {
+    if bb.utils.to_boolean(d.getVar('CACHE_DEB_SRC')):
+        bb.build.addtask('do_cache_deb_src', 'do_deploy', 'do_image', d)
+}
+
 
 inherit rootfs
 inherit sdk
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 205da640..b9d360e7 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -272,7 +272,8 @@ python do_rootfs_install() {
 }
 addtask rootfs_install before do_rootfs_postprocess after do_unpack
 
-cache_deb_src() {
+do_cache_deb_src[network] = "${TASK_USE_SUDO}"
+do_cache_deb_src() {
     if [ -e "${ROOTFSDIR}"/etc/resolv.conf ] ||
        [ -h "${ROOTFSDIR}"/etc/resolv.conf ]; then
         sudo mv "${ROOTFSDIR}"/etc/resolv.conf "${ROOTFSDIR}"/etc/resolv.conf.isar
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250305142513.1841415-2-srinuvasan.a%40siemens.com.

[PATCH 3/3] image: disable clean-log-files ROOTFS_FEATURES for deb-src caching

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

By default the /var/log/dpkg.log file is removed as part of
rootfs postprocessing in the regular image builds, but this is
needed to get the IMAGE_ROOTFS related deb-src packages during debian source caching,
hence make this feature like a configurable one rather than enabled by default.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/classes/image.bbclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 66b4d313..94ecef86 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,8 @@ inherit multiarch
 inherit essential
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "clean-package-cache clean-pycache generate-manifest export-dpkg-status clean-log-files clean-debconf-cache"
+ROOTFS_FEATURES += "clean-package-cache clean-pycache generate-manifest export-dpkg-status clean-debconf-cache"
+ROOTFS_FEATURES += "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', '', 'clean-log-files', d)}"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${@isar_multiarch_packages('IMAGE_INSTALL', d)}"
 ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250305142513.1841415-3-srinuvasan.a%40siemens.com.

Re: [PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Jan Kiszka' via isar-users]

On 05.03.25 15:25, srinuvasan.a via isar-users wrote:
> From: srinuvasan <srinuvasan.a@siemens.com>
> 
> Without this option, mmdebstrap deletes the var/log/dpkg.log and its siblings by default,
> but this is needed to download deb-src packages for base-apt creation.
> Basically, we are referring to the package's status under /var/log/dpkg.log file
> and performing the debsrc_download operation.
> 
> Without these changes, bootstrap related deb-src files are not downloaded
> under the /downloads/deb-src folder, due to this the repo sanity check
> functions failed during base-apt caching stage.
> 
> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> ---
>  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> index 931f6f13..7380a658 100644
> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -168,6 +168,11 @@ do_bootstrap() {
>          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
>      fi
>  
> +    cleanup=
> +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
> +        cleanup="--skip=cleanup/reproducible"
> +    fi
> +

Uhh, this is a nasty dependency. We must document this and warn the user
that the generated images are now longer reproducible.

Or am I misinterpreting the impact?

Jan

>      # Cleanup mounts if fails
>      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
>      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
> @@ -183,6 +188,7 @@ do_bootstrap() {
>  
>      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
>                     $arch_param \
> +                   $cleanup \
>                     --mode=unshare \
>                     ${MMHOOKS} \
>                     --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \

-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/6a7cf2f0-d826-467a-b85e-2fd666bb92e7%40siemens.com.

Re: [PATCH 2/3] image: download deb-src packages after imager creation

[Uladzimir Bely]

On Wed, 2025-03-05 at 19:55 +0530, srinuvasan.a@siemens.com wrote:
> From: srinuvasan <srinuvasan.a@siemens.com>
> 
> With the present implementation, deb-src packages are missing for
> imager related
> deb packages (packages listed in IMAGER_INSTALL).
> Copy the /var/log/dpkg.log file from the sbuild environment to
> outside
> With that, use the list to download deb-src packages for
> IMAGER_INSTALL
> related deb packages.
> 
> Presently, cache_deb_src function running as part of
> ROOTFS_POSTPROCESS_COMMAND,
> this will not be helpful to get the imager related deb-src packages,
> hence
> make cache_deb_src as a separate task and running
> always after do_image to get the imager deps related deb-src
> packages.
> 
> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> ---
>  meta/classes/image-tools-extension.bbclass | 8 +++++++-
>  meta/classes/image.bbclass                 | 8 +++++++-
>  meta/classes/rootfs.bbclass                | 3 ++-
>  3 files changed, 16 insertions(+), 3 deletions(-)
> 
> diff --git a/meta/classes/image-tools-extension.bbclass
> b/meta/classes/image-tools-extension.bbclass
> index e8ace8f5..156dfcb3 100644
> --- a/meta/classes/image-tools-extension.bbclass
> +++ b/meta/classes/image-tools-extension.bbclass
> @@ -7,6 +7,9 @@
>  
>  inherit sbuild
>  
> +PP = "/home/builder/${PN}-${MACHINE}"
> +ext_root="${PP}/rootfs"
> +
>  IMAGER_INSTALL ??= ""
>  IMAGER_BUILD_DEPS ??= ""
>  DEPENDS += "${IMAGER_BUILD_DEPS}"
> @@ -59,9 +62,11 @@ EOAPT
>  
>          deb_dl_dir_export ${schroot_dir} ${distro}
>          schroot -r -c ${session_id} -d / -u root -- sh -c " \
> +            rm -f /var/log/dpkg.log
>              apt-get -o Debug::pkgProblemResolver=yes --no-install-
> recommends -y \
>                  --allow-unauthenticated --allow-downgrades install \
> -                ${local_install}"
> +                ${local_install}
> +            cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log"

This fails on any build with FSTYPES including (at least) "ext4":

```
[stdlog] 2025-03-05 19:12:45,460 avocado.app ERROR| ERROR: Logfile of
failure stored in:
/workspace/build/isar_ub_devel_fast/210/build/tmp/work/debian-bullseye-
armhf/isar-image-base-qemuarm/1.0-r0/temp/log.do_image_ext4.3684960
...
[stdlog] 2025-03-05 19:12:54,914 avocado.test INFO | | cp: cannot stat
'/var/log/dpkg.log': No such file or directory
```

Can be easy reproduced with kas menu, if "ext4" is added to additional
image fstypes.

>      fi
>  
>      schroot -r -c ${session_id} "$@"
> @@ -70,4 +75,5 @@ EOAPT
>  
>      remove_mounts
>      schroot_delete_configs
> +    sbuild_dpkg_log_export "${WORKDIR}/rootfs/dpkg_partial.log"
>  }
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index 56eca202..66b4d313 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -72,7 +72,13 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?=
> "${DEPLOY_DIR_IMAGE}"
>  ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
>  ROOTFS_PACKAGE_SUFFIX ?= "${PN}-${DISTRO}-${MACHINE}"
>  
> -ROOTFS_POSTPROCESS_COMMAND:prepend =
> "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src',
> 'cache_deb_src', '', d)} "
> +CACHE_DEB_SRC = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-
> deb-src', '1', '0', d)}"
> +
> +python () {
> +    if bb.utils.to_boolean(d.getVar('CACHE_DEB_SRC')):
> +        bb.build.addtask('do_cache_deb_src', 'do_deploy',
> 'do_image', d)
> +}
> +
>  
>  inherit rootfs
>  inherit sdk
> diff --git a/meta/classes/rootfs.bbclass
> b/meta/classes/rootfs.bbclass
> index 205da640..b9d360e7 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -272,7 +272,8 @@ python do_rootfs_install() {
>  }
>  addtask rootfs_install before do_rootfs_postprocess after do_unpack
>  
> -cache_deb_src() {
> +do_cache_deb_src[network] = "${TASK_USE_SUDO}"
> +do_cache_deb_src() {
>      if [ -e "${ROOTFSDIR}"/etc/resolv.conf ] ||
>         [ -h "${ROOTFSDIR}"/etc/resolv.conf ]; then
>          sudo mv "${ROOTFSDIR}"/etc/resolv.conf
> "${ROOTFSDIR}"/etc/resolv.conf.isar

-- 
Best regards,
Uladzimir.

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/d58ee313adf4bf1f289b8d8c5c89a58c6fcc4e0f.camel%40ilbers.de.

RE: [PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Arjunan, Srinu' via isar-users]

Hi Jan,

           Yes with this option the images are reproducible, I don’t' find other option to enable /var/log/dpkg.log in mmdebstrap, will capture in the documentation.

Thanks,
Srinu

-----Original Message-----
From: Kiszka, Jan (FT RPD CED) <jan.kiszka@siemens.com> 
Sent: 05 March 2025 20:10
To: Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>; isar-users@googlegroups.com
Cc: Hombourger, Cedric (FT FDS CES LX) <cedric.hombourger@siemens.com>; ubely@ilbers.de; Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com>
Subject: Re: [PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

On 05.03.25 15:25, srinuvasan.a via isar-users wrote:
> From: srinuvasan <srinuvasan.a@siemens.com>
> 
> Without this option, mmdebstrap deletes the var/log/dpkg.log and its 
> siblings by default, but this is needed to download deb-src packages for base-apt creation.
> Basically, we are referring to the package's status under 
> /var/log/dpkg.log file and performing the debsrc_download operation.
> 
> Without these changes, bootstrap related deb-src files are not 
> downloaded under the /downloads/deb-src folder, due to this the repo 
> sanity check functions failed during base-apt caching stage.
> 
> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> ---
>  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
> b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> index 931f6f13..7380a658 100644
> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -168,6 +168,11 @@ do_bootstrap() {
>          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
>      fi
>  
> +    cleanup=
> +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
> +        cleanup="--skip=cleanup/reproducible"
> +    fi
> +

Uhh, this is a nasty dependency. We must document this and warn the user that the generated images are now longer reproducible.

Or am I misinterpreting the impact?

Jan

>      # Cleanup mounts if fails
>      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
>      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat 
> "${WORKDIR}/mmtmpdir") \ @@ -183,6 +188,7 @@ do_bootstrap() {
>  
>      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
>                     $arch_param \
> +                   $cleanup \
>                     --mode=unshare \
>                     ${MMHOOKS} \
>                     --setup-hook='mkdir -p 
> "$1/var/cache/apt/archives/"' \

--
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/PUZPR06MB5747A370F04BAFBB173E30F188CA2%40PUZPR06MB5747.apcprd06.prod.outlook.com.

RE: [PATCH 2/3] image: download deb-src packages after imager creation

['Arjunan, Srinu' via isar-users]

Thanks Uladzimir for the findings, will try to reproduce and fix the issue.

Thanks,
Srinu

-----Original Message-----
From: Uladzimir Bely <ubely@ilbers.de> 
Sent: 06 March 2025 11:35
To: Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>; isar-users@googlegroups.com
Subject: Re: [PATCH 2/3] image: download deb-src packages after imager creation

On Wed, 2025-03-05 at 19:55 +0530, srinuvasan.a@siemens.com wrote:
> From: srinuvasan <srinuvasan.a@siemens.com>
> 
> With the present implementation, deb-src packages are missing for 
> imager related deb packages (packages listed in IMAGER_INSTALL).
> Copy the /var/log/dpkg.log file from the sbuild environment to outside 
> With that, use the list to download deb-src packages for 
> IMAGER_INSTALL related deb packages.
> 
> Presently, cache_deb_src function running as part of 
> ROOTFS_POSTPROCESS_COMMAND, this will not be helpful to get the imager 
> related deb-src packages, hence make cache_deb_src as a separate task 
> and running always after do_image to get the imager deps related 
> deb-src packages.
> 
> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> ---
>  meta/classes/image-tools-extension.bbclass | 8 +++++++-
>  meta/classes/image.bbclass                 | 8 +++++++-
>  meta/classes/rootfs.bbclass                | 3 ++-
>  3 files changed, 16 insertions(+), 3 deletions(-)
> 
> diff --git a/meta/classes/image-tools-extension.bbclass
> b/meta/classes/image-tools-extension.bbclass
> index e8ace8f5..156dfcb3 100644
> --- a/meta/classes/image-tools-extension.bbclass
> +++ b/meta/classes/image-tools-extension.bbclass
> @@ -7,6 +7,9 @@
>  
>  inherit sbuild
>  
> +PP = "/home/builder/${PN}-${MACHINE}"
> +ext_root="${PP}/rootfs"
> +
>  IMAGER_INSTALL ??= ""
>  IMAGER_BUILD_DEPS ??= ""
>  DEPENDS += "${IMAGER_BUILD_DEPS}"
> @@ -59,9 +62,11 @@ EOAPT
>  
>          deb_dl_dir_export ${schroot_dir} ${distro}
>          schroot -r -c ${session_id} -d / -u root -- sh -c " \
> +            rm -f /var/log/dpkg.log
>              apt-get -o Debug::pkgProblemResolver=yes --no-install- 
> recommends -y \
>                  --allow-unauthenticated --allow-downgrades install \
> -                ${local_install}"
> +                ${local_install}
> +            cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log"

This fails on any build with FSTYPES including (at least) "ext4":

```
[stdlog] 2025-03-05 19:12:45,460 avocado.app ERROR| ERROR: Logfile of failure stored in:
/workspace/build/isar_ub_devel_fast/210/build/tmp/work/debian-bullseye-
armhf/isar-image-base-qemuarm/1.0-r0/temp/log.do_image_ext4.3684960
...
[stdlog] 2025-03-05 19:12:54,914 avocado.test INFO | | cp: cannot stat
'/var/log/dpkg.log': No such file or directory ```

Can be easy reproduced with kas menu, if "ext4" is added to additional image fstypes.

>      fi
>  
>      schroot -r -c ${session_id} "$@"
> @@ -70,4 +75,5 @@ EOAPT
>  
>      remove_mounts
>      schroot_delete_configs
> +    sbuild_dpkg_log_export "${WORKDIR}/rootfs/dpkg_partial.log"
>  }
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass 
> index 56eca202..66b4d313 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -72,7 +72,13 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
>  ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
>  ROOTFS_PACKAGE_SUFFIX ?= "${PN}-${DISTRO}-${MACHINE}"
>  
> -ROOTFS_POSTPROCESS_COMMAND:prepend =
> "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 
> 'cache_deb_src', '', d)} "
> +CACHE_DEB_SRC = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-
> deb-src', '1', '0', d)}"
> +
> +python () {
> +    if bb.utils.to_boolean(d.getVar('CACHE_DEB_SRC')):
> +        bb.build.addtask('do_cache_deb_src', 'do_deploy',
> 'do_image', d)
> +}
> +
>  
>  inherit rootfs
>  inherit sdk
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass 
> index 205da640..b9d360e7 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -272,7 +272,8 @@ python do_rootfs_install() {
>  }
>  addtask rootfs_install before do_rootfs_postprocess after do_unpack
>  
> -cache_deb_src() {
> +do_cache_deb_src[network] = "${TASK_USE_SUDO}"
> +do_cache_deb_src() {
>      if [ -e "${ROOTFSDIR}"/etc/resolv.conf ] ||
>         [ -h "${ROOTFSDIR}"/etc/resolv.conf ]; then
>          sudo mv "${ROOTFSDIR}"/etc/resolv.conf 
> "${ROOTFSDIR}"/etc/resolv.conf.isar

--
Best regards,
Uladzimir.

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/PUZPR06MB5747ADC9174D4F1FAD6B990888CA2%40PUZPR06MB5747.apcprd06.prod.outlook.com.

Re: [PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Jan Kiszka' via isar-users]

On 06.03.25 07:08, Arjunan, Srinu (FT FDS CES LX PBU 2) wrote:
> Hi Jan,
> 
>            Yes with this option the images are reproducible, I don’t' find other option to enable /var/log/dpkg.log in mmdebstrap, will capture in the documentation.

Wait: Images are *still* reproducibe, despite skipping the
"cleanup/reproducible" step in mmdebstrap? Even if so, we cannot be sure
that future changes to mmdebstrap would not break this again.

Jan

> 
> Thanks,
> Srinu
> 
> -----Original Message-----
> From: Kiszka, Jan (FT RPD CED) <jan.kiszka@siemens.com> 
> Sent: 05 March 2025 20:10
> To: Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>; isar-users@googlegroups.com
> Cc: Hombourger, Cedric (FT FDS CES LX) <cedric.hombourger@siemens.com>; ubely@ilbers.de; Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com>
> Subject: Re: [PATCH 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap
> 
> On 05.03.25 15:25, srinuvasan.a via isar-users wrote:
>> From: srinuvasan <srinuvasan.a@siemens.com>
>>
>> Without this option, mmdebstrap deletes the var/log/dpkg.log and its 
>> siblings by default, but this is needed to download deb-src packages for base-apt creation.
>> Basically, we are referring to the package's status under 
>> /var/log/dpkg.log file and performing the debsrc_download operation.
>>
>> Without these changes, bootstrap related deb-src files are not 
>> downloaded under the /downloads/deb-src folder, due to this the repo 
>> sanity check functions failed during base-apt caching stage.
>>
>> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
>> ---
>>  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
>>  1 file changed, 6 insertions(+)
>>
>> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
>> b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
>> index 931f6f13..7380a658 100644
>> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
>> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
>> @@ -168,6 +168,11 @@ do_bootstrap() {
>>          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
>>      fi
>>  
>> +    cleanup=
>> +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
>> +        cleanup="--skip=cleanup/reproducible"
>> +    fi
>> +
> 
> Uhh, this is a nasty dependency. We must document this and warn the user that the generated images are now longer reproducible.
> 
> Or am I misinterpreting the impact?
> 
> Jan
> 
>>      # Cleanup mounts if fails
>>      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
>>      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat 
>> "${WORKDIR}/mmtmpdir") \ @@ -183,6 +188,7 @@ do_bootstrap() {
>>  
>>      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
>>                     $arch_param \
>> +                   $cleanup \
>>                     --mode=unshare \
>>                     ${MMHOOKS} \
>>                     --setup-hook='mkdir -p 
>> "$1/var/cache/apt/archives/"' \
> 
> --
> Siemens AG, Foundational Technologies
> Linux Expert Center


-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/b7758b1c-a2b9-4ba4-928b-d44f67580273%40siemens.com.

[PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

Without this option, mmdebstrap deletes the var/log/dpkg.log and its siblings by default,
but this is needed to download deb-src packages for base-apt creation.
Basically, we are referring to the package's status under /var/log/dpkg.log file
and performing the debsrc_download operation.

Without these changes, bootstrap related deb-src files are not downloaded
under the /downloads/deb-src folder, due to this the repo sanity check
functions failed during base-apt caching stage.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 931f6f13..7380a658 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -168,6 +168,11 @@ do_bootstrap() {
         arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
     fi
 
+    cleanup=
+    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
+        cleanup="--skip=cleanup/reproducible"
+    fi
+
     # Cleanup mounts if fails
     trap 'exit 1' INT HUP QUIT TERM ALRM USR1
     trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
@@ -183,6 +188,7 @@ do_bootstrap() {
 
     sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
                    $arch_param \
+                   $cleanup \
                    --mode=unshare \
                    ${MMHOOKS} \
                    --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250307122558.1882902-1-srinuvasan.a%40siemens.com.

[PATCH v2 2/3] image: download deb-src packages after imager creation

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

With the present implementation, deb-src packages are missing for imager related
deb packages (packages listed in IMAGER_INSTALL).
Copy the /var/log/dpkg.log file from the sbuild environment to outside
With that, use the list to download deb-src packages for IMAGER_INSTALL
related deb packages.

Presently, cache_deb_src function running as part of ROOTFS_POSTPROCESS_COMMAND,
this will not be helpful to get the imager related deb-src packages, hence
make cache_deb_src as a separate task and running
always after do_image to get the imager deps related deb-src packages.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/classes/image-tools-extension.bbclass | 19 +++++++++++++++++--
 meta/classes/image.bbclass                 |  8 +++++++-
 meta/classes/rootfs.bbclass                |  3 ++-
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index e8ace8f5..0615229a 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -7,6 +7,9 @@
 
 inherit sbuild
 
+PP = "/home/builder/${PN}-${MACHINE}"
+ext_root="${PP}/rootfs"
+
 IMAGER_INSTALL ??= ""
 IMAGER_BUILD_DEPS ??= ""
 DEPENDS += "${IMAGER_BUILD_DEPS}"
@@ -58,10 +61,17 @@ imager_run() {
 EOAPT
 
         deb_dl_dir_export ${schroot_dir} ${distro}
-        schroot -r -c ${session_id} -d / -u root -- sh -c " \
+        export local_install=${local_install}
+        schroot -r -c ${session_id} -d / -u root -- /bin/bash <<"EOAPT"
+            rm -f /var/log/dpkg.log
+            pkg_status="$(dpkg-query --showformat='${binary:Package}\t' --show ${local_install} || echo "Not installed")"
             apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
                 --allow-unauthenticated --allow-downgrades install \
-                ${local_install}"
+                ${local_install}
+            if echo "${pkg_status}" | grep  "Not installed"; then
+                cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log
+            fi
+EOAPT
     fi
 
     schroot -r -c ${session_id} "$@"
@@ -69,5 +79,10 @@ EOAPT
     schroot -e -c ${session_id}
 
     remove_mounts
+
     schroot_delete_configs
+
+    if [ -e ${WORKDIR}/rootfs/dpkg_partial.log ]; then
+        sbuild_dpkg_log_export "${WORKDIR}/rootfs/dpkg_partial.log"
+    fi
 }
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 56eca202..66b4d313 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -72,7 +72,13 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_PACKAGE_SUFFIX ?= "${PN}-${DISTRO}-${MACHINE}"
 
-ROOTFS_POSTPROCESS_COMMAND:prepend = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'cache_deb_src', '', d)} "
+CACHE_DEB_SRC = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', '1', '0', d)}"
+
+python () {
+    if bb.utils.to_boolean(d.getVar('CACHE_DEB_SRC')):
+        bb.build.addtask('do_cache_deb_src', 'do_deploy', 'do_image', d)
+}
+
 
 inherit rootfs
 inherit sdk
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 2348e269..c9e832dc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -272,7 +272,8 @@ python do_rootfs_install() {
 }
 addtask rootfs_install before do_rootfs_postprocess after do_unpack
 
-cache_deb_src() {
+do_cache_deb_src[network] = "${TASK_USE_SUDO}"
+do_cache_deb_src() {
     if [ -e "${ROOTFSDIR}"/etc/resolv.conf ] ||
        [ -h "${ROOTFSDIR}"/etc/resolv.conf ]; then
         sudo mv "${ROOTFSDIR}"/etc/resolv.conf "${ROOTFSDIR}"/etc/resolv.conf.isar
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250307122558.1882902-2-srinuvasan.a%40siemens.com.

[PATCH v2 3/3] image: disable clean-log-files ROOTFS_FEATURES for deb-src caching

[srinuvasan.a via isar-users]

From: srinuvasan <srinuvasan.a@siemens.com>

By default the /var/log/dpkg.log file is removed as part of
rootfs postprocessing in the regular image builds, but this is
needed to get the IMAGE_ROOTFS related deb-src packages during debian source caching,
hence make this feature like a configurable one rather than enabled by default.

Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
---
 meta/classes/image.bbclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 66b4d313..94ecef86 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,8 @@ inherit multiarch
 inherit essential
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "clean-package-cache clean-pycache generate-manifest export-dpkg-status clean-log-files clean-debconf-cache"
+ROOTFS_FEATURES += "clean-package-cache clean-pycache generate-manifest export-dpkg-status clean-debconf-cache"
+ROOTFS_FEATURES += "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', '', 'clean-log-files', d)}"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${@isar_multiarch_packages('IMAGE_INSTALL', d)}"
 ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 ROOTFS_DPKGSTATUS_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
-- 
2.39.5

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250307122558.1882902-3-srinuvasan.a%40siemens.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Jan Kiszka' via isar-users]

On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> From: srinuvasan <srinuvasan.a@siemens.com>
> 
> Without this option, mmdebstrap deletes the var/log/dpkg.log and its siblings by default,
> but this is needed to download deb-src packages for base-apt creation.
> Basically, we are referring to the package's status under /var/log/dpkg.log file
> and performing the debsrc_download operation.
> 
> Without these changes, bootstrap related deb-src files are not downloaded
> under the /downloads/deb-src folder, due to this the repo sanity check
> functions failed during base-apt caching stage.
> 
> Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> ---
>  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> index 931f6f13..7380a658 100644
> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -168,6 +168,11 @@ do_bootstrap() {
>          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
>      fi
>  
> +    cleanup=
> +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]; then
> +        cleanup="--skip=cleanup/reproducible"
> +    fi
> +
>      # Cleanup mounts if fails
>      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
>      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
> @@ -183,6 +188,7 @@ do_bootstrap() {
>  
>      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
>                     $arch_param \
> +                   $cleanup \
>                     --mode=unshare \
>                     ${MMHOOKS} \
>                     --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \

What has changed in this version? I'm not seeing my concerns addressed,
my questions answered.

Jan

-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/47d50bdd-2f10-4373-9009-554365643775%40siemens.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['cedric.hombourger@siemens.com' via isar-users]

On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > From: srinuvasan <srinuvasan.a@siemens.com>
> > 
> > Without this option, mmdebstrap deletes the var/log/dpkg.log and
> > its siblings by default,
> > but this is needed to download deb-src packages for base-apt
> > creation.

I don't see where deb-src needs dpkg.log. Can you shed some light?
Both the old code and the code I recently changed parse the .deb files
that were downloaded to determine which source packages should be
downloaded

Ref: https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y

> > Basically, we are referring to the package's status under
> > /var/log/dpkg.log file
> > and performing the debsrc_download operation.
> > 
> > Without these changes, bootstrap related deb-src files are not
> > downloaded
> > under the /downloads/deb-src folder, due to this the repo sanity
> > check
> > functions failed during base-apt caching stage.

What are these repo sanity checks?
Where do we see the failures?

From my base-apt workdir:

pwd && grep -ir error
<...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-r0/temp
run.repo.7287:		echo "ERROR: $*"

We probably need to be more specific if we are mentioning a failure.
Without this, people on this list will not understand what problem(s)
you are fixing

> > 
> > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > ---
> >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
> >  1 file changed, 6 insertions(+)
> > 
> > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > index 931f6f13..7380a658 100644
> > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > @@ -168,6 +168,11 @@ do_bootstrap() {
> >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> >      fi
> >  
> > +    cleanup=
> > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-
> > src', 'yes', 'no', d)}" = "yes" ]; then
> > +        cleanup="--skip=cleanup/reproducible"
> > +    fi
> > +
> >      # Cleanup mounts if fails
> >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > "${WORKDIR}/mmtmpdir") \
> > @@ -183,6 +188,7 @@ do_bootstrap() {
> >  
> >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
> >                     $arch_param \
> > +                   $cleanup \
> >                     --mode=unshare \
> >                     ${MMHOOKS} \
> >                     --setup-hook='mkdir -p
> > "$1/var/cache/apt/archives/"' \
> 
> What has changed in this version? I'm not seeing my concerns
> addressed,
> my questions answered.

+1 

Please make sure new revisions of your patch series include a changelog
(use --cover-letter with git format-patch)

> 
> Jan
> 


-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/06c431535ff06f5f854f8f846dd511db1c5c82be.camel%40siemens.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

[Srinuvasan Arjunan]

[-- Attachment #1.1: Type: text/plain, Size: 2530 bytes --]



On Saturday, March 8, 2025 at 3:30:04 AM UTC+5:30 Jan Kiszka wrote:

On 07.03.25 13:25, srinuv...@siemens.com wrote: 
> From: srinuvasan <srinuv...@siemens.com> 
> 
> Without this option, mmdebstrap deletes the var/log/dpkg.log and its 
siblings by default, 
> but this is needed to download deb-src packages for base-apt creation. 
> Basically, we are referring to the package's status under 
/var/log/dpkg.log file 
> and performing the debsrc_download operation. 
> 
> Without these changes, bootstrap related deb-src files are not downloaded 
> under the /downloads/deb-src folder, due to this the repo sanity check 
> functions failed during base-apt caching stage. 
> 
> Signed-off-by: srinuvasan <srinuv...@siemens.com> 
> --- 
> meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++ 
> 1 file changed, 6 insertions(+) 
> 
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
> index 931f6f13..7380a658 100644 
> --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc 
> @@ -168,6 +168,11 @@ do_bootstrap() { 
> arch_param="$arch_param,${COMPAT_DISTRO_ARCH}" 
> fi 
> 
> + cleanup= 
> + if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 
'yes', 'no', d)}" = "yes" ]; then 
> + cleanup="--skip=cleanup/reproducible" 
> + fi 
> + 
> # Cleanup mounts if fails 
> trap 'exit 1' INT HUP QUIT TERM ALRM USR1 
> trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") 
\ 
> @@ -183,6 +188,7 @@ do_bootstrap() { 
> 
> sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \ 
> $arch_param \ 
> + $cleanup \ 
> --mode=unshare \ 
> ${MMHOOKS} \ 
> --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \ 

What has changed in this version? I'm not seeing my concerns addressed, 
my questions answered.


  Hi jan,

   I will come up with the more details for your query, this version 
address the failures reported by Uladzimir Bely. 



Jan 

-- 
Siemens AG, Foundational Technologies 
Linux Expert Center 

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/1a6be420-acd6-4f40-94bc-058fe954f55dn%40googlegroups.com.

[-- Attachment #1.2: Type: text/html, Size: 3753 bytes --]

RE: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Arjunan, Srinu' via isar-users]

-----Original Message-----
From: Hombourger, Cedric (FT FDS CES LX) <cedric.hombourger@siemens.com>
Sent: 08 March 2025 19:15
To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>
Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
Subject: Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > From: srinuvasan <srinuvasan.a@siemens.com>
> >
> > Without this option, mmdebstrap deletes the var/log/dpkg.log and its
> > siblings by default, but this is needed to download deb-src packages
> > for base-apt creation.

I don't see where deb-src needs dpkg.log. Can you shed some light?
Both the old code and the code I recently changed parse the .deb files that were downloaded to determine which source packages should be downloaded

Ref: https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y

> > Basically, we are referring to the package's status under
> > /var/log/dpkg.log file and performing the debsrc_download operation.
> >
> > Without these changes, bootstrap related deb-src files are not
> > downloaded under the /downloads/deb-src folder, due to this the repo
> > sanity check functions failed during base-apt caching stage.

What are these repo sanity checks?
Where do we see the failures?

From my base-apt workdir:

pwd && grep -ir error
<...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-r0/temp
run.repo.7287:          echo "ERROR: $*"

We probably need to be more specific if we are mentioning a failure.
Without this, people on this list will not understand what problem(s) you are fixing


Hi cedric, jan

I will explain the issue in the details below:

                To reproduce the issue ( Ref: https://github.com/ilbers/isar/blob/master/doc/user_manual.md#creation-of-local-apt-repo-caching-upstream-debian-packages ):

                Trigger the warmup build to download deb and deb-src packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in conf/local.conf)

                e.x: bitbake mc:qemuamd64-bookworm:isar-image-base

                Once all the deb and deb-src downloaded for the particular BSP under /download folder

                Trigger the base-apt target to cache all the deb and deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)

                e.x: bitbake base-apt

                You will see the failures:

                Logs:

                NOTE: Resolving any missing task queue dependencies
Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0% match, 0% complete)#############################################            | ETA:  0:00:00
Initialising tasks: 100% |####################################################################################################################| Time: 0:00:00
NOTE: No setscene tasks
NOTE: Executing Tasks
ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in repo. bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-2_all.deb
bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb
bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian-archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-1_all.deb
bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find-rule-perl/libfile-find-rule-perl_0.34-3_all.deb
bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber-compare-perl/libnumber-compare-perl_0.03-3_all.deb
bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-6+deb12u1_all.deb
bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-common_3.4-1_all.deb
bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob-perl/libtext-glob-perl_0.11-3_all.deb
bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules-5.36_5.36.0-7+deb12u1_all.deb
bookworm six 1.16.0-4 pool/main/s/six/python3-six_1.16.0-4_all.deb
bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-0+deb12u1_all.deb
bookworm usrmerge 37~deb12u1 pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb

Just i captured few packages, but many of the deb packages don't have their source files

In the base-apt recipes we are calling the repo_sanity_test function finally to make sure all the deb's having
the corresponding deb-src packages should be present in the base-apt.


Yes, as you said, we are parsing deb files (from /var/cache/apt/archives) that were downloaded to determine which source packages should be downloaded.
but in the function "is_not_part_of_current_build "${package}" we are skipping the download-src if parsing deb package (status installed) is not available in the below files
```
            "${IMAGE_ROOTFS}"/var/log/dpkg.log \
            "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
            "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
            "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
            "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
```

As I mentioned above, the adduser package was installed in the stage of bootstrap, but that package's status was not captured in the bootstrap's /var/log/dpkg.log file.
because mmdebstrap deletes those files by default, hence we didn't capture those bootstrap package status.

As we know bootstrap rootfs is used for sbuildchroot creation, but here whatever packages are installed during sbuild-chroot creation,
${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the sbuildchroot related package status, but our expectation is it should have
bootstrap + sbuildchroot package status available in /var/log/dpkg.log file in sbuildchroot.

My first patch solves the above issue
with the introduction of "skip=cleanup/reproducible", the bootstrap rootfs generated with /var/log/dpkg.log files, which contain all the bootstrap package status
along with that if you use that boostrap for sbuildchroot creation , the sbuildchroot rootfs contains the existing bootstrap plus
additionally installed sbuildchroot related packages status is available in sbuldchroot /var/log/dpkg.log file


My second patch resolve the below issue:

During image creation (IMAGE_FSTYPES = "wic") few packages were installed in the session (e.g: gdisk, bmap-tools) but those packages list available
only in the sbuild environment, not outside, carry those packages status to tmp/dpkg_common.log in the rootfs to get the src packages for
imager_install related deb packages, presently the cache-deb-src function calls in the ROOTFS_POSTPROCESS_COMMAND but we need to wait till imager generation
That's why we pushed that deb-src-caching after image creation.

My third path resolve the image rootfs related deb-src missing issue:

By default we are removing the /var/log/dpkg.log file in the rootfs_postprocess_clean_log_files function, but this is needed when
We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making this function as a configurable one rather than enabled by default.
to download IMAGE_ROOTFS related deb-src packages

I hope I explain the issue, presently am checking the reproducible part (for jan queries), will update that part very soon.

Many thanks,
Srinu

> >
> > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > ---
> >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
> >  1 file changed, 6 insertions(+)
> >
> > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > index 931f6f13..7380a658 100644
> > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > @@ -168,6 +168,11 @@ do_bootstrap() {
> >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> >      fi
> >
> > +    cleanup=
> > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-
> > src', 'yes', 'no', d)}" = "yes" ]; then
> > +        cleanup="--skip=cleanup/reproducible"
> > +    fi
> > +
> >      # Cleanup mounts if fails
> >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > "${WORKDIR}/mmtmpdir") \
> > @@ -183,6 +188,7 @@ do_bootstrap() {
> >
> >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
> >                     $arch_param \
> > +                   $cleanup \
> >                     --mode=unshare \
> >                     ${MMHOOKS} \
> >                     --setup-hook='mkdir -p
> > "$1/var/cache/apt/archives/"' \
>
> What has changed in this version? I'm not seeing my concerns
> addressed, my questions answered.

+1

Please make sure new revisions of your patch series include a changelog (use --cover-letter with git format-patch)

Sure

>
> Jan
>


-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/PUZPR06MB57479349B38D83094DD2821A88D42%40PUZPR06MB5747.apcprd06.prod.outlook.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['cedric.hombourger@siemens.com' via isar-users]

On Sat, 2025-03-08 at 18:25 +0000, Arjunan, Srinu (FT FDS CES LX PBU 2)
wrote:
>
>
> -----Original Message-----
> From: Hombourger, Cedric (FT FDS CES LX)
> <cedric.hombourger@siemens.com>
> Sent: 08 March 2025 19:15
> To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED)
> <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2)
> <srinuvasan.a@siemens.com>
> Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> Subject: Re: [PATCH v2 1/3] mmdebstrap: add
> "skip=cleanup/reproducible" option to mmdebstrap
>
> On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> > On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > > From: srinuvasan <srinuvasan.a@siemens.com>
> > >
> > > Without this option, mmdebstrap deletes the var/log/dpkg.log and
> > > its
> > > siblings by default, but this is needed to download deb-src
> > > packages
> > > for base-apt creation.
>
> I don't see where deb-src needs dpkg.log. Can you shed some light?
> Both the old code and the code I recently changed parse the .deb
> files that were downloaded to determine which source packages should
> be downloaded
>
> Ref:
> https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y
>
> > > Basically, we are referring to the package's status under
> > > /var/log/dpkg.log file and performing the debsrc_download
> > > operation.
> > >
> > > Without these changes, bootstrap related deb-src files are not
> > > downloaded under the /downloads/deb-src folder, due to this the
> > > repo
> > > sanity check functions failed during base-apt caching stage.
>
> What are these repo sanity checks?
> Where do we see the failures?
>
> From my base-apt workdir:
>
> pwd && grep -ir error
> <...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-
> r0/temp
> run.repo.7287:          echo "ERROR: $*"
>
> We probably need to be more specific if we are mentioning a failure.
> Without this, people on this list will not understand what problem(s)
> you are fixing
>
>
> Hi cedric, jan
>
> I will explain the issue in the details below:
>
>                 To reproduce the issue ( Ref:
> https://github.com/ilbers/isar/blob/master/doc/user_manual.md#creation-of-local-apt-repo-caching-upstream-debian-packages
>  ):
>
>                 Trigger the warmup build to download deb and deb-src
> packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in
> conf/local.conf)
>
>                 e.x: bitbake mc:qemuamd64-bookworm:isar-image-base
>
>                 Once all the deb and deb-src downloaded for the
> particular BSP under /download folder
>
>                 Trigger the base-apt target to cache all the deb and
> deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)
>
>                 e.x: bitbake base-apt
>
>                 You will see the failures:
>
>                 Logs:
>
>                 NOTE: Resolving any missing task queue dependencies
> Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0%
> match, 0%
> complete)#############################################            |
> ETA:  0:00:00
> Initialising tasks: 100%
> |####################################################################
> ################################################| Time: 0:00:00
> NOTE: No setscene tasks
> NOTE: Executing Tasks
> ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in
> repo. bookworm adduser 3.134
> pool/main/a/adduser/adduser_3.134_all.deb
> bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-
> 2_all.deb
> bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb
> bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian-
> archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
> bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-
> 1_all.deb
> bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find-
> rule-perl/libfile-find-rule-perl_0.34-3_all.deb
> bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber-
> compare-perl/libnumber-compare-perl_0.03-3_all.deb
> bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-
> 6+deb12u1_all.deb

.dsc for pam packages found in our downstream project

> bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-
> common_3.4-1_all.deb
> bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob-
> perl/libtext-glob-perl_0.11-3_all.deb
> bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules-
> 5.36_5.36.0-7+deb12u1_all.deb
> bookworm six 1.16.0-4 pool/main/s/six/python3-six_1.16.0-4_all.deb
> bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-
> 0+deb12u1_all.deb
> bookworm usrmerge 37~deb12u1
> pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
> bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
>
> Just i captured few packages, but many of the deb packages don't have
> their source files
>
> In the base-apt recipes we are calling the repo_sanity_test function
> finally to make sure all the deb's having
> the corresponding deb-src packages should be present in the base-apt.

I checked again log.do_cache in our downstream project and I did not
see any of the mentioned errors.

$ grep -v 'Export' ./tmp/work/debian-bookworm-any/base-apt/1.0-
r0/temp/log.do_cache
DEBUG: Executing python function do_cache
DEBUG: Using absolute /build/../repo/gnupg/205D601E.pub
DEBUG: Executing shell function repo
DEBUG: Shell function repo finished
DEBUG: Python function do_cache finished

That build was however done with the patch I have submitted earlier to
this list. It also includes our own layer. I will do a plain Isar build
(with my patch) to check if the issue is reproducible.

I took a few error cases from your example above and verified that the
.dsc files are present in the generated base-apt build.

It should be noted that my patch removes
"is_not_part_of_current_build()"

parsing dpkg.log was IMO not a great idea as I don't think its format
is officially documented (I could be wrong though)

>
>
> Yes, as you said, we are parsing deb files (from
> /var/cache/apt/archives) that were downloaded to determine which
> source packages should be downloaded.
> but in the function "is_not_part_of_current_build "${package}" we are
> skipping the download-src if parsing deb package (status installed)
> is not available in the below files
> ```
>             "${IMAGE_ROOTFS}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
>             "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
> ```
>
> As I mentioned above, the adduser package was installed in the stage
> of bootstrap, but that package's status was not captured in the
> bootstrap's /var/log/dpkg.log file.
> because mmdebstrap deletes those files by default, hence we didn't
> capture those bootstrap package status.
>
> As we know bootstrap rootfs is used for sbuildchroot creation, but
> here whatever packages are installed during sbuild-chroot creation,
> ${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the
> sbuildchroot related package status, but our expectation is it should
> have
> bootstrap + sbuildchroot package status available in
> /var/log/dpkg.log file in sbuildchroot.
>
> My first patch solves the above issue
> with the introduction of "skip=cleanup/reproducible", the bootstrap
> rootfs generated with /var/log/dpkg.log files, which contain all the
> bootstrap package status
> along with that if you use that boostrap for sbuildchroot creation ,
> the sbuildchroot rootfs contains the existing bootstrap plus
> additionally installed sbuildchroot related packages status is
> available in sbuldchroot /var/log/dpkg.log file
>
>
> My second patch resolve the below issue:
>
> During image creation (IMAGE_FSTYPES = "wic") few packages were
> installed in the session (e.g: gdisk, bmap-tools) but those packages
> list available
> only in the sbuild environment, not outside, carry those packages
> status to tmp/dpkg_common.log in the rootfs to get the src packages
> for
> imager_install related deb packages, presently the cache-deb-src
> function calls in the ROOTFS_POSTPROCESS_COMMAND but we need to wait
> till imager generation
> That's why we pushed that deb-src-caching after image creation.
>
> My third path resolve the image rootfs related deb-src missing issue:
>
> By default we are removing the /var/log/dpkg.log file in the
> rootfs_postprocess_clean_log_files function, but this is needed when
> We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making this
> function as a configurable one rather than enabled by default.
> to download IMAGE_ROOTFS related deb-src packages
>
> I hope I explain the issue, presently am checking the reproducible
> part (for jan queries), will update that part very soon.
>
> Many thanks,
> Srinu
>
> > >
> > > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > > ---
> > >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
> > >  1 file changed, 6 insertions(+)
> > >
> > > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-
> > > mmdebstrap.inc
> > > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > index 931f6f13..7380a658 100644
> > > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > @@ -168,6 +168,11 @@ do_bootstrap() {
> > >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> > >      fi
> > >
> > > +    cleanup=
> > > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-
> > > src', 'yes', 'no', d)}" = "yes" ]; then
> > > +        cleanup="--skip=cleanup/reproducible"
> > > +    fi
> > > +
> > >      # Cleanup mounts if fails
> > >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> > >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > > "${WORKDIR}/mmtmpdir") \
> > > @@ -183,6 +188,7 @@ do_bootstrap() {
> > >
> > >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args
> > > \
> > >                     $arch_param \
> > > +                   $cleanup \
> > >                     --mode=unshare \
> > >                     ${MMHOOKS} \
> > >                     --setup-hook='mkdir -p
> > > "$1/var/cache/apt/archives/"' \
> >
> > What has changed in this version? I'm not seeing my concerns
> > addressed, my questions answered.
>
> +1
>
> Please make sure new revisions of your patch series include a
> changelog (use --cover-letter with git format-patch)
>
> Sure
>
> >
> > Jan
> >
>
>

--
Cedric Hombourger
Siemens AG
http://www.siemens.com/

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/c0d69628cbbc8dcfed0fa80f30225cc9f683f1d1.camel%40siemens.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['cedric.hombourger@siemens.com' via isar-users]

On Sat, 2025-03-08 at 18:25 +0000, Arjunan, Srinu (FT FDS CES LX PBU 2)
wrote:
>
>
> -----Original Message-----
> From: Hombourger, Cedric (FT FDS CES LX)
> <cedric.hombourger@siemens.com>
> Sent: 08 March 2025 19:15
> To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED)
> <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2)
> <srinuvasan.a@siemens.com>
> Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> Subject: Re: [PATCH v2 1/3] mmdebstrap: add
> "skip=cleanup/reproducible" option to mmdebstrap
>
> On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> > On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > > From: srinuvasan <srinuvasan.a@siemens.com>
> > >
> > > Without this option, mmdebstrap deletes the var/log/dpkg.log and
> > > its
> > > siblings by default, but this is needed to download deb-src
> > > packages
> > > for base-apt creation.
>
> I don't see where deb-src needs dpkg.log. Can you shed some light?
> Both the old code and the code I recently changed parse the .deb
> files that were downloaded to determine which source packages should
> be downloaded
>
> Ref:
> https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y
>
> > > Basically, we are referring to the package's status under
> > > /var/log/dpkg.log file and performing the debsrc_download
> > > operation.
> > >
> > > Without these changes, bootstrap related deb-src files are not
> > > downloaded under the /downloads/deb-src folder, due to this the
> > > repo
> > > sanity check functions failed during base-apt caching stage.
>
> What are these repo sanity checks?
> Where do we see the failures?
>
> From my base-apt workdir:
>
> pwd && grep -ir error
> <...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-
> r0/temp
> run.repo.7287:          echo "ERROR: $*"
>
> We probably need to be more specific if we are mentioning a failure.
> Without this, people on this list will not understand what problem(s)
> you are fixing
>
>
> Hi cedric, jan
>
> I will explain the issue in the details below:
>
>                 To reproduce the issue ( Ref:
> https://github.com/ilbers/isar/blob/master/doc/user_manual.md#creation-of-local-apt-repo-caching-upstream-debian-packages
>  ):
>
>                 Trigger the warmup build to download deb and deb-src
> packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in
> conf/local.conf)
>
>                 e.x: bitbake mc:qemuamd64-bookworm:isar-image-base
>
>                 Once all the deb and deb-src downloaded for the
> particular BSP under /download folder
>
>                 Trigger the base-apt target to cache all the deb and
> deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)
>
>                 e.x: bitbake base-apt
>
>                 You will see the failures:
>
>                 Logs:
>
>                 NOTE: Resolving any missing task queue dependencies
> Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0%
> match, 0%
> complete)#############################################            |
> ETA:  0:00:00
> Initialising tasks: 100%
> |####################################################################
> ################################################| Time: 0:00:00
> NOTE: No setscene tasks
> NOTE: Executing Tasks
> ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in
> repo. bookworm adduser 3.134
> pool/main/a/adduser/adduser_3.134_all.deb

.dsc found in our downstream project

> bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-
> 2_all.deb
> bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb

.dsc found in our downstream project

> bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian-
> archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
> bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-
> 1_all.deb
> bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find-
> rule-perl/libfile-find-rule-perl_0.34-3_all.deb
> bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber-
> compare-perl/libnumber-compare-perl_0.03-3_all.deb
> bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-
> 6+deb12u1_all.deb


> bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-
> common_3.4-1_all.deb
> bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob-
> perl/libtext-glob-perl_0.11-3_all.deb
> bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules-
> 5.36_5.36.0-7+deb12u1_all.deb
> bookworm six 1.16.0-4 pool/main/s/six/python3-six_1.16.0-4_all.deb
> bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-
> 0+deb12u1_all.deb
> bookworm usrmerge 37~deb12u1
> pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
> bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
>
> Just i captured few packages, but many of the deb packages don't have
> their source files
>
> In the base-apt recipes we are calling the repo_sanity_test function
> finally to make sure all the deb's having
> the corresponding deb-src packages should be present in the base-apt.
>
>
> Yes, as you said, we are parsing deb files (from
> /var/cache/apt/archives) that were downloaded to determine which
> source packages should be downloaded.
> but in the function "is_not_part_of_current_build "${package}" we are
> skipping the download-src if parsing deb package (status installed)
> is not available in the below files
> ```
>             "${IMAGE_ROOTFS}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
>             "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
> ```
>
> As I mentioned above, the adduser package was installed in the stage
> of bootstrap, but that package's status was not captured in the
> bootstrap's /var/log/dpkg.log file.
> because mmdebstrap deletes those files by default, hence we didn't
> capture those bootstrap package status.
>
> As we know bootstrap rootfs is used for sbuildchroot creation, but
> here whatever packages are installed during sbuild-chroot creation,
> ${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the
> sbuildchroot related package status, but our expectation is it should
> have
> bootstrap + sbuildchroot package status available in
> /var/log/dpkg.log file in sbuildchroot.
>
> My first patch solves the above issue
> with the introduction of "skip=cleanup/reproducible", the bootstrap
> rootfs generated with /var/log/dpkg.log files, which contain all the
> bootstrap package status
> along with that if you use that boostrap for sbuildchroot creation ,
> the sbuildchroot rootfs contains the existing bootstrap plus
> additionally installed sbuildchroot related packages status is
> available in sbuldchroot /var/log/dpkg.log file
>
>
> My second patch resolve the below issue:
>
> During image creation (IMAGE_FSTYPES = "wic") few packages were
> installed in the session (e.g: gdisk, bmap-tools) but those packages
> list available
> only in the sbuild environment, not outside, carry those packages
> status to tmp/dpkg_common.log in the rootfs to get the src packages
> for
> imager_install related deb packages, presently the cache-deb-src
> function calls in the ROOTFS_POSTPROCESS_COMMAND but we need to wait
> till imager generation
> That's why we pushed that deb-src-caching after image creation.
>
> My third path resolve the image rootfs related deb-src missing issue:
>
> By default we are removing the /var/log/dpkg.log file in the
> rootfs_postprocess_clean_log_files function, but this is needed when
> We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making this
> function as a configurable one rather than enabled by default.
> to download IMAGE_ROOTFS related deb-src packages
>
> I hope I explain the issue, presently am checking the reproducible
> part (for jan queries), will update that part very soon.
>
> Many thanks,
> Srinu
>
> > >
> > > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > > ---
> > >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
> > >  1 file changed, 6 insertions(+)
> > >
> > > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-
> > > mmdebstrap.inc
> > > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > index 931f6f13..7380a658 100644
> > > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > @@ -168,6 +168,11 @@ do_bootstrap() {
> > >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> > >      fi
> > >
> > > +    cleanup=
> > > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-
> > > src', 'yes', 'no', d)}" = "yes" ]; then
> > > +        cleanup="--skip=cleanup/reproducible"
> > > +    fi
> > > +
> > >      # Cleanup mounts if fails
> > >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> > >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > > "${WORKDIR}/mmtmpdir") \
> > > @@ -183,6 +188,7 @@ do_bootstrap() {
> > >
> > >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args
> > > \
> > >                     $arch_param \
> > > +                   $cleanup \
> > >                     --mode=unshare \
> > >                     ${MMHOOKS} \
> > >                     --setup-hook='mkdir -p
> > > "$1/var/cache/apt/archives/"' \
> >
> > What has changed in this version? I'm not seeing my concerns
> > addressed, my questions answered.
>
> +1
>
> Please make sure new revisions of your patch series include a
> changelog (use --cover-letter with git format-patch)
>
> Sure
>
> >
> > Jan
> >
>
>

--
Cedric Hombourger
Siemens AG
http://www.siemens.com/

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/64884fdb07b7d44ed1f676bdfeddc9b5135be417.camel%40siemens.com.

RE: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Arjunan, Srinu' via isar-users]

-----Original Message-----
From: Hombourger, Cedric (FT FDS CES LX) <cedric.hombourger@siemens.com> 
Sent: 09 March 2025 00:46
To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>
Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
Subject: Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

On Sat, 2025-03-08 at 18:25 +0000, Arjunan, Srinu (FT FDS CES LX PBU 2)
wrote:
> 
> 
> -----Original Message-----
> From: Hombourger, Cedric (FT FDS CES LX) 
> <cedric.hombourger@siemens.com>
> Sent: 08 March 2025 19:15
> To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) 
> <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) 
> <srinuvasan.a@siemens.com>
> Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> Subject: Re: [PATCH v2 1/3] mmdebstrap: add 
> "skip=cleanup/reproducible" option to mmdebstrap
> 
> On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> > On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > > From: srinuvasan <srinuvasan.a@siemens.com>
> > > 
> > > Without this option, mmdebstrap deletes the var/log/dpkg.log and 
> > > its siblings by default, but this is needed to download deb-src 
> > > packages for base-apt creation.
> 
> I don't see where deb-src needs dpkg.log. Can you shed some light?
> Both the old code and the code I recently changed parse the .deb files 
> that were downloaded to determine which source packages should be 
> downloaded
> 
> Ref:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgrou
> ps.google.com%2Fg%2Fisar-users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Csrinuv
> asan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae3bcd9579
> 4fd4addab42e1495d55a%7C1%7C0%7C638770581452433346%7CUnknown%7CTWFpbGZs
> b3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIj
> oiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=b49oZTatP9h6CxEys%2BPv%2
> F%2F%2F5v2BmE%2FX1%2FdhDPs%2F1qOY%3D&reserved=0
> 
> > > Basically, we are referring to the package's status under 
> > > /var/log/dpkg.log file and performing the debsrc_download 
> > > operation.
> > > 
> > > Without these changes, bootstrap related deb-src files are not 
> > > downloaded under the /downloads/deb-src folder, due to this the 
> > > repo sanity check functions failed during base-apt caching stage.
> 
> What are these repo sanity checks?
> Where do we see the failures?
> 
> From my base-apt workdir:
> 
> pwd && grep -ir error
> <...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-
> r0/temp
> run.repo.7287:          echo "ERROR: $*"
> 
> We probably need to be more specific if we are mentioning a failure.
> Without this, people on this list will not understand what problem(s) 
> you are fixing
> 
> 
> Hi cedric, jan
> 
> I will explain the issue in the details below:
> 
>                 To reproduce the issue ( Ref:
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgith
> ub.com%2Filbers%2Fisar%2Fblob%2Fmaster%2Fdoc%2Fuser_manual.md%23creati
> on-of-local-apt-repo-caching-upstream-debian-packages&data=05%7C02%7Cs
> rinuvasan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae3bc
> d95794fd4addab42e1495d55a%7C1%7C0%7C638770581452457810%7CUnknown%7CTWF
> pbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsI
> kFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=3XId9a7IuBK9xyzYWnu
> dmeG0aKe3awlNSGSmwId7D%2BU%3D&reserved=0
>  ):
> 
>                 Trigger the warmup build to download deb and deb-src 
> packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in
> conf/local.conf)
> 
>                 e.x: bitbake mc:qemuamd64-bookworm:isar-image-base
> 
>                 Once all the deb and deb-src downloaded for the 
> particular BSP under /download folder
> 
>                 Trigger the base-apt target to cache all the deb and 
> deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)
> 
>                 e.x: bitbake base-apt
> 
>                 You will see the failures:
>                 
>                 Logs: 
>                 
>                 NOTE: Resolving any missing task queue dependencies 
> Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0% 
> match, 0% complete)#############################################            
> |
> ETA:  0:00:00
> Initialising tasks: 100%
> |####################################################################
> ################################################| Time: 0:00:00
> NOTE: No setscene tasks
> NOTE: Executing Tasks
> ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in 
> repo. bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
> bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-
> 2_all.deb
> bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb
> bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian- 
> archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
> bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-
> 1_all.deb
> bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find- 
> rule-perl/libfile-find-rule-perl_0.34-3_all.deb
> bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber- 
> compare-perl/libnumber-compare-perl_0.03-3_all.deb
> bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-
> 6+deb12u1_all.deb

.dsc for pam packages found in our downstream project

Yes cedric, validated from my end, it is available in downstream project because we have your changes in isar-patches (https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y)

But still we can reproduce the issue in vanilla ISAR without your patch, I started working on these changes before you proposed these changes (https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y)

Basically, your patches remove the is_not_part_of_current_build() function
And refactor the debsrc_download functionality.

 My patch becomes obsoleted if your (https://groups.google.com/g/isar-users/c/OwrSBmCPe0Y) changes are going to merge in ISAR.

Please provide your thoughts?

> bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-
> common_3.4-1_all.deb
> bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob- 
> perl/libtext-glob-perl_0.11-3_all.deb
> bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules- 
> 5.36_5.36.0-7+deb12u1_all.deb bookworm six 1.16.0-4 
> pool/main/s/six/python3-six_1.16.0-4_all.deb
> bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-
> 0+deb12u1_all.deb
> bookworm usrmerge 37~deb12u1
> pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
> bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
> 
> Just i captured few packages, but many of the deb packages don't have 
> their source files
> 
> In the base-apt recipes we are calling the repo_sanity_test function 
> finally to make sure all the deb's having the corresponding deb-src 
> packages should be present in the base-apt.

I checked again log.do_cache in our downstream project and I did not see any of the mentioned errors.

In downstream project we are not at all calling the repo_sanity_test function due to enabled the cache-deb-src in ROOTFS_POSTPROCESS_COMMAND rather than enabled in conf/local.conf

The below condition fails:
if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ]

that is the reason we are not calling this statement at all " reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing "

because we are not enabled the cache-deb-src via BASE_REPO_FEATURES


$ grep -v 'Export' ./tmp/work/debian-bookworm-any/base-apt/1.0-
r0/temp/log.do_cache
DEBUG: Executing python function do_cache
DEBUG: Using absolute /build/../repo/gnupg/205D601E.pub
DEBUG: Executing shell function repo
DEBUG: Shell function repo finished
DEBUG: Python function do_cache finished

That build was however done with the patch I have submitted earlier to this list. It also includes our own layer. I will do a plain Isar build (with my patch) to check if the issue is reproducible.

I took a few error cases from your example above and verified that the .dsc files are present in the generated base-apt build.

It should be noted that my patch removes "is_not_part_of_current_build()"

parsing dpkg.log was IMO not a great idea as I don't think its format is officially documented (I could be wrong though)

> 
>                 
> Yes, as you said, we are parsing deb files (from
> /var/cache/apt/archives) that were downloaded to determine which 
> source packages should be downloaded.
> but in the function "is_not_part_of_current_build "${package}" we are 
> skipping the download-src if parsing deb package (status installed) is 
> not available in the below files ```
>             "${IMAGE_ROOTFS}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
>             "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
>             "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
> ```
> 
> As I mentioned above, the adduser package was installed in the stage 
> of bootstrap, but that package's status was not captured in the 
> bootstrap's /var/log/dpkg.log file.
> because mmdebstrap deletes those files by default, hence we didn't 
> capture those bootstrap package status.
> 
> As we know bootstrap rootfs is used for sbuildchroot creation, but 
> here whatever packages are installed during sbuild-chroot creation, 
> ${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the 
> sbuildchroot related package status, but our expectation is it should 
> have bootstrap + sbuildchroot package status available in 
> /var/log/dpkg.log file in sbuildchroot.
> 
> My first patch solves the above issue
> with the introduction of "skip=cleanup/reproducible", the bootstrap 
> rootfs generated with /var/log/dpkg.log files, which contain all the 
> bootstrap package status along with that if you use that boostrap for 
> sbuildchroot creation , the sbuildchroot rootfs contains the existing 
> bootstrap plus additionally installed sbuildchroot related packages 
> status is available in sbuldchroot /var/log/dpkg.log file
> 
> 
> My second patch resolve the below issue:
> 
> During image creation (IMAGE_FSTYPES = "wic") few packages were 
> installed in the session (e.g: gdisk, bmap-tools) but those packages 
> list available only in the sbuild environment, not outside, carry 
> those packages status to tmp/dpkg_common.log in the rootfs to get the 
> src packages for imager_install related deb packages, presently the 
> cache-deb-src function calls in the ROOTFS_POSTPROCESS_COMMAND but we 
> need to wait till imager generation That's why we pushed that 
> deb-src-caching after image creation.
> 
> My third path resolve the image rootfs related deb-src missing issue:
> 
> By default we are removing the /var/log/dpkg.log file in the 
> rootfs_postprocess_clean_log_files function, but this is needed when 
> We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making this 
> function as a configurable one rather than enabled by default.
> to download IMAGE_ROOTFS related deb-src packages
> 
> I hope I explain the issue, presently am checking the reproducible 
> part (for jan queries), will update that part very soon.
> 
> Many thanks,
> Srinu
> 
> > > 
> > > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > > ---
> > >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6 ++++++
> > >  1 file changed, 6 insertions(+)
> > > 
> > > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-
> > > mmdebstrap.inc
> > > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > index 931f6f13..7380a658 100644
> > > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > @@ -168,6 +168,11 @@ do_bootstrap() {
> > >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> > >      fi
> > >  
> > > +    cleanup=
> > > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-
> > > src', 'yes', 'no', d)}" = "yes" ]; then
> > > +        cleanup="--skip=cleanup/reproducible"
> > > +    fi
> > > +
> > >      # Cleanup mounts if fails
> > >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> > >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > > "${WORKDIR}/mmtmpdir") \
> > > @@ -183,6 +188,7 @@ do_bootstrap() {
> > >  
> > >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args 
> > > \
> > >                     $arch_param \
> > > +                   $cleanup \
> > >                     --mode=unshare \
> > >                     ${MMHOOKS} \
> > >                     --setup-hook='mkdir -p 
> > > "$1/var/cache/apt/archives/"' \
> > 
> > What has changed in this version? I'm not seeing my concerns 
> > addressed, my questions answered.
> 
> +1
> 
> Please make sure new revisions of your patch series include a 
> changelog (use --cover-letter with git format-patch)
> 
> Sure
> 
> > 
> > Jan
> > 
> 
> 

--
Cedric Hombourger
Siemens AG
www.siemens.com

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/PUZPR06MB5747ED47D64E89DE5D9A70B388D72%40PUZPR06MB5747.apcprd06.prod.outlook.com.

Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['cedric.hombourger@siemens.com' via isar-users]

On Sun, 2025-03-09 at 08:46 +0000, Arjunan, Srinu (FT FDS CES LX PBU 2)
wrote:
> 
> 
> -----Original Message-----
> From: Hombourger, Cedric (FT FDS CES LX)
> <cedric.hombourger@siemens.com> 
> Sent: 09 March 2025 00:46
> To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED)
> <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2)
> <srinuvasan.a@siemens.com>
> Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> Subject: Re: [PATCH v2 1/3] mmdebstrap: add
> "skip=cleanup/reproducible" option to mmdebstrap
> 
> On Sat, 2025-03-08 at 18:25 +0000, Arjunan, Srinu (FT FDS CES LX PBU
> 2)
> wrote:
> > 
> > 
> > -----Original Message-----
> > From: Hombourger, Cedric (FT FDS CES LX) 
> > <cedric.hombourger@siemens.com>
> > Sent: 08 March 2025 19:15
> > To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) 
> > <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) 
> > <srinuvasan.a@siemens.com>
> > Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> > Subject: Re: [PATCH v2 1/3] mmdebstrap: add 
> > "skip=cleanup/reproducible" option to mmdebstrap
> > 
> > On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> > > On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > > > From: srinuvasan <srinuvasan.a@siemens.com>
> > > > 
> > > > Without this option, mmdebstrap deletes the var/log/dpkg.log
> > > > and 
> > > > its siblings by default, but this is needed to download deb-src
> > > > packages for base-apt creation.
> > 
> > I don't see where deb-src needs dpkg.log. Can you shed some light?
> > Both the old code and the code I recently changed parse the .deb
> > files 
> > that were downloaded to determine which source packages should be 
> > downloaded
> > 
> > Ref:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgrou%2F&data=05%7C02%7Ccedric.hombourger%40siemens.com%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771068307563689%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=kPyQeXBw6pYTZng1OieFPozM1xn9YgRdyuC%2FSfGK7o0%3D&reserved=0
> > ps.google.com%2Fg%2Fisar-
> > users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Csrinuv
> > asan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae3bcd9
> > 579
> > 4fd4addab42e1495d55a%7C1%7C0%7C638770581452433346%7CUnknown%7CTWFpb
> > GZs
> > b3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkF
> > OIj
> > oiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=b49oZTatP9h6CxEys%2BP
> > v%2
> > F%2F%2F5v2BmE%2FX1%2FdhDPs%2F1qOY%3D&reserved=0
> > 
> > > > Basically, we are referring to the package's status under 
> > > > /var/log/dpkg.log file and performing the debsrc_download 
> > > > operation.
> > > > 
> > > > Without these changes, bootstrap related deb-src files are not 
> > > > downloaded under the /downloads/deb-src folder, due to this the
> > > > repo sanity check functions failed during base-apt caching
> > > > stage.
> > 
> > What are these repo sanity checks?
> > Where do we see the failures?
> > 
> > From my base-apt workdir:
> > 
> > pwd && grep -ir error
> > <...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-
> > r0/temp
> > run.repo.7287:          echo "ERROR: $*"
> > 
> > We probably need to be more specific if we are mentioning a
> > failure.
> > Without this, people on this list will not understand what
> > problem(s) 
> > you are fixing
> > 
> > 
> > Hi cedric, jan
> > 
> > I will explain the issue in the details below:
> > 
> >                 To reproduce the issue ( Ref:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgith%2F&data=05%7C02%7Ccedric.hombourger%40siemens.com%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771068307586848%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=AUH81pXJwMNzrIIkLA3GkJ2fLWwL9jBFGeltINWR41g%3D&reserved=0
> > ub.com%2Filbers%2Fisar%2Fblob%2Fmaster%2Fdoc%2Fuser_manual.md%23cre
> > ati
> > on-of-local-apt-repo-caching-upstream-debian-
> > packages&data=05%7C02%7Cs
> > rinuvasan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae
> > 3bc
> > d95794fd4addab42e1495d55a%7C1%7C0%7C638770581452457810%7CUnknown%7C
> > TWF
> > pbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMi
> > IsI
> > kFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=3XId9a7IuBK9xyzY
> > Wnu
> > dmeG0aKe3awlNSGSmwId7D%2BU%3D&reserved=0
> >  ):
> > 
> >                 Trigger the warmup build to download deb and deb-
> > src 
> > packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in
> > conf/local.conf)
> > 
> >                 e.x: bitbake mc:qemuamd64-bookworm:isar-image-base
> > 
> >                 Once all the deb and deb-src downloaded for the 
> > particular BSP under /download folder
> > 
> >                 Trigger the base-apt target to cache all the deb
> > and 
> > deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)
> > 
> >                 e.x: bitbake base-apt
> > 
> >                 You will see the failures:
> >                 
> >                 Logs: 
> >                 
> >                 NOTE: Resolving any missing task queue dependencies
> > Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0% 
> > match, 0%
> > complete)#############################################            
> > > 
> > ETA:  0:00:00
> > Initialising tasks: 100%
> > > #################################################################
> > > ###
> > ################################################| Time: 0:00:00
> > NOTE: No setscene tasks
> > NOTE: Executing Tasks
> > ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in
> > repo. bookworm adduser 3.134
> > pool/main/a/adduser/adduser_3.134_all.deb
> > bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-
> > 2_all.deb
> > bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb
> > bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian- 
> > archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
> > bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-
> > 1_all.deb
> > bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find-
> > rule-perl/libfile-find-rule-perl_0.34-3_all.deb
> > bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber- 
> > compare-perl/libnumber-compare-perl_0.03-3_all.deb
> > bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-
> > 6+deb12u1_all.deb
> 
> .dsc for pam packages found in our downstream project
> 
> Yes cedric, validated from my end, it is available in downstream
> project because we have your changes in isar-patches
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307601863%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=OoUVTbJLnDaPdXKYmZmOh0Wxn9HpiRis%2FYRaL
> YL2Khs%3D&reserved=0)
> 
> But still we can reproduce the issue in vanilla ISAR without your
> patch, I started working on these changes before you proposed these
> changes
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307616236%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=IknRwnuSOZiNUPnx43d4W4dJd2vmylKoUyp%2Bp
> T4W6OE%3D&reserved=0)
> 
> Basically, your patches remove the is_not_part_of_current_build()
> function
> And refactor the debsrc_download functionality.
> 
>  My patch becomes obsoleted if your
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307630537%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=vkjpy98ZFZZo4WgKjtwj7UREcd%2B2J4EBcWliJ
> eIdHbU%3D&reserved=0) changes are going to merge in ISAR.
> 
> Please provide your thoughts?


Well the future of my patch and yours are more for the Isar maintainers
to decide. They definitely conflict with each other.

> 
> > bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-
> > common_3.4-1_all.deb
> > bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob- 
> > perl/libtext-glob-perl_0.11-3_all.deb
> > bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules- 
> > 5.36_5.36.0-7+deb12u1_all.deb bookworm six 1.16.0-4 
> > pool/main/s/six/python3-six_1.16.0-4_all.deb
> > bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-
> > 0+deb12u1_all.deb
> > bookworm usrmerge 37~deb12u1
> > pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
> > bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
> > 
> > Just i captured few packages, but many of the deb packages don't
> > have 
> > their source files
> > 
> > In the base-apt recipes we are calling the repo_sanity_test
> > function 
> > finally to make sure all the deb's having the corresponding deb-src
> > packages should be present in the base-apt.
> 
> I checked again log.do_cache in our downstream project and I did not
> see any of the mentioned errors.
> 
> In downstream project we are not at all calling the repo_sanity_test
> function due to enabled the cache-deb-src in
> ROOTFS_POSTPROCESS_COMMAND rather than enabled in conf/local.conf
> 
> The below condition fails:
> if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src',
> 'yes', 'no', d)}" = "yes" ]
> 
> that is the reason we are not calling this statement at all "
> reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing "
> 
> because we are not enabled the cache-deb-src via BASE_REPO_FEATURES
> 
> 
> $ grep -v 'Export' ./tmp/work/debian-bookworm-any/base-apt/1.0-
> r0/temp/log.do_cache
> DEBUG: Executing python function do_cache
> DEBUG: Using absolute /build/../repo/gnupg/205D601E.pub
> DEBUG: Executing shell function repo
> DEBUG: Shell function repo finished
> DEBUG: Python function do_cache finished
> 
> That build was however done with the patch I have submitted earlier
> to this list. It also includes our own layer. I will do a plain Isar
> build (with my patch) to check if the issue is reproducible.
> 
> I took a few error cases from your example above and verified that
> the .dsc files are present in the generated base-apt build.
> 
> It should be noted that my patch removes
> "is_not_part_of_current_build()"
> 
> parsing dpkg.log was IMO not a great idea as I don't think its format
> is officially documented (I could be wrong though)
> 
> > 
> >                 
> > Yes, as you said, we are parsing deb files (from
> > /var/cache/apt/archives) that were downloaded to determine which 
> > source packages should be downloaded.
> > but in the function "is_not_part_of_current_build "${package}" we
> > are 
> > skipping the download-src if parsing deb package (status installed)
> > is 
> > not available in the below files ```
> >             "${IMAGE_ROOTFS}"/var/log/dpkg.log \
> >             "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
> >             "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
> >             "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
> >             "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
> > ```
> > 
> > As I mentioned above, the adduser package was installed in the
> > stage 
> > of bootstrap, but that package's status was not captured in the 
> > bootstrap's /var/log/dpkg.log file.
> > because mmdebstrap deletes those files by default, hence we didn't 
> > capture those bootstrap package status.
> > 
> > As we know bootstrap rootfs is used for sbuildchroot creation, but 
> > here whatever packages are installed during sbuild-chroot creation,
> > ${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the
> > sbuildchroot related package status, but our expectation is it
> > should 
> > have bootstrap + sbuildchroot package status available in 
> > /var/log/dpkg.log file in sbuildchroot.
> > 
> > My first patch solves the above issue
> > with the introduction of "skip=cleanup/reproducible", the bootstrap
> > rootfs generated with /var/log/dpkg.log files, which contain all
> > the 
> > bootstrap package status along with that if you use that boostrap
> > for 
> > sbuildchroot creation , the sbuildchroot rootfs contains the
> > existing 
> > bootstrap plus additionally installed sbuildchroot related packages
> > status is available in sbuldchroot /var/log/dpkg.log file
> > 
> > 
> > My second patch resolve the below issue:
> > 
> > During image creation (IMAGE_FSTYPES = "wic") few packages were 
> > installed in the session (e.g: gdisk, bmap-tools) but those
> > packages 
> > list available only in the sbuild environment, not outside, carry 
> > those packages status to tmp/dpkg_common.log in the rootfs to get
> > the 
> > src packages for imager_install related deb packages, presently the
> > cache-deb-src function calls in the ROOTFS_POSTPROCESS_COMMAND but
> > we 
> > need to wait till imager generation That's why we pushed that 
> > deb-src-caching after image creation.
> > 
> > My third path resolve the image rootfs related deb-src missing
> > issue:
> > 
> > By default we are removing the /var/log/dpkg.log file in the 
> > rootfs_postprocess_clean_log_files function, but this is needed
> > when 
> > We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making
> > this 
> > function as a configurable one rather than enabled by default.
> > to download IMAGE_ROOTFS related deb-src packages
> > 
> > I hope I explain the issue, presently am checking the reproducible 
> > part (for jan queries), will update that part very soon.
> > 
> > Many thanks,
> > Srinu
> > 
> > > > 
> > > > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > > > ---
> > > >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6
> > > > ++++++
> > > >  1 file changed, 6 insertions(+)
> > > > 
> > > > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-
> > > > mmdebstrap.inc
> > > > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > index 931f6f13..7380a658 100644
> > > > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > @@ -168,6 +168,11 @@ do_bootstrap() {
> > > >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> > > >      fi
> > > >  
> > > > +    cleanup=
> > > > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-
> > > > deb-
> > > > src', 'yes', 'no', d)}" = "yes" ]; then
> > > > +        cleanup="--skip=cleanup/reproducible"
> > > > +    fi
> > > > +
> > > >      # Cleanup mounts if fails
> > > >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> > > >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > > > "${WORKDIR}/mmtmpdir") \
> > > > @@ -183,6 +188,7 @@ do_bootstrap() {
> > > >  
> > > >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap
> > > > $bootstrap_args 
> > > > \
> > > >                     $arch_param \
> > > > +                   $cleanup \
> > > >                     --mode=unshare \
> > > >                     ${MMHOOKS} \
> > > >                     --setup-hook='mkdir -p 
> > > > "$1/var/cache/apt/archives/"' \
> > > 
> > > What has changed in this version? I'm not seeing my concerns 
> > > addressed, my questions answered.
> > 
> > +1
> > 
> > Please make sure new revisions of your patch series include a 
> > changelog (use --cover-letter with git format-patch)
> > 
> > Sure
> > 
> > > 
> > > Jan
> > > 
> > 
> > 
> 
> --
> Cedric Hombourger
> Siemens AG
> www.siemens.com

-- 
Cedric Hombourger
Siemens AG
www.siemens.com

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/ac437554de180e0465e53fd3266684e822875bbd.camel%40siemens.com.

RE: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

['Arjunan, Srinu' via isar-users]

-----Original Message-----
From: Hombourger, Cedric (FT FDS CES LX) <cedric.hombourger@siemens.com> 
Sent: 10 March 2025 14:45
To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) <srinuvasan.a@siemens.com>
Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
Subject: Re: [PATCH v2 1/3] mmdebstrap: add "skip=cleanup/reproducible" option to mmdebstrap

On Sun, 2025-03-09 at 08:46 +0000, Arjunan, Srinu (FT FDS CES LX PBU 2)
wrote:
> 
> 
> -----Original Message-----
> From: Hombourger, Cedric (FT FDS CES LX) 
> <cedric.hombourger@siemens.com>
> Sent: 09 March 2025 00:46
> To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) 
> <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) 
> <srinuvasan.a@siemens.com>
> Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> Subject: Re: [PATCH v2 1/3] mmdebstrap: add 
> "skip=cleanup/reproducible" option to mmdebstrap
> 
> On Sat, 2025-03-08 at 18:25 +0000, Arjunan, Srinu (FT FDS CES LX PBU
> 2)
> wrote:
> > 
> > 
> > -----Original Message-----
> > From: Hombourger, Cedric (FT FDS CES LX) 
> > <cedric.hombourger@siemens.com>
> > Sent: 08 March 2025 19:15
> > To: isar-users@googlegroups.com; Kiszka, Jan (FT RPD CED) 
> > <jan.kiszka@siemens.com>; Arjunan, Srinu (FT FDS CES LX PBU 2) 
> > <srinuvasan.a@siemens.com>
> > Cc: ubely@ilbers.de; Adithya.Balakumar@toshiba-tsip.com
> > Subject: Re: [PATCH v2 1/3] mmdebstrap: add 
> > "skip=cleanup/reproducible" option to mmdebstrap
> > 
> > On Fri, 2025-03-07 at 22:59 +0100, Jan Kiszka wrote:
> > > On 07.03.25 13:25, srinuvasan.a@siemens.com wrote:
> > > > From: srinuvasan <srinuvasan.a@siemens.com>
> > > > 
> > > > Without this option, mmdebstrap deletes the var/log/dpkg.log and 
> > > > its siblings by default, but this is needed to download deb-src 
> > > > packages for base-apt creation.
> > 
> > I don't see where deb-src needs dpkg.log. Can you shed some light?
> > Both the old code and the code I recently changed parse the .deb 
> > files that were downloaded to determine which source packages should 
> > be downloaded
> > 
> > Ref:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr
> > ou%2F&data=05%7C02%7Csrinuvasan.a%40siemens.com%7Ccd17f90bd20b4d46a1
> > ff08dd5fb403e6%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C63877194
> > 8975549635%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwL
> > jAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7
> > C%7C&sdata=oekcbPth33aUQlgM3jGv9ukM1%2BnxgNW1y51931942zE%3D&reserved
> > =0
> > ps.google.com%2Fg%2Fisar-
> > users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Csrinuv
> > asan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae3bcd9
> > 579
> > 4fd4addab42e1495d55a%7C1%7C0%7C638770581452433346%7CUnknown%7CTWFpb
> > GZs
> > b3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkF
> > OIj
> > oiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=b49oZTatP9h6CxEys%2BP
> > v%2
> > F%2F%2F5v2BmE%2FX1%2FdhDPs%2F1qOY%3D&reserved=0
> > 
> > > > Basically, we are referring to the package's status under 
> > > > /var/log/dpkg.log file and performing the debsrc_download 
> > > > operation.
> > > > 
> > > > Without these changes, bootstrap related deb-src files are not 
> > > > downloaded under the /downloads/deb-src folder, due to this the 
> > > > repo sanity check functions failed during base-apt caching 
> > > > stage.
> > 
> > What are these repo sanity checks?
> > Where do we see the failures?
> > 
> > From my base-apt workdir:
> > 
> > pwd && grep -ir error
> > <...>/build/base-apt/tmp/work/debian-bookworm-any/base-apt/1.0-
> > r0/temp
> > run.repo.7287:          echo "ERROR: $*"
> > 
> > We probably need to be more specific if we are mentioning a failure.
> > Without this, people on this list will not understand what
> > problem(s)
> > you are fixing
> > 
> > 
> > Hi cedric, jan
> > 
> > I will explain the issue in the details below:
> > 
> >                 To reproduce the issue ( Ref:
> > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgith%2F&data=05%7C02%7Csrinuvasan.a%40siemens.com%7Ccd17f90bd20b4d46a1ff08dd5fb403e6%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771948975572755%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=iTA44lkFK0i58FlzXMRLIDZ5bfgeHTk2cDgv2Q%2BVogU%3D&reserved=0
> > ub.com%2Filbers%2Fisar%2Fblob%2Fmaster%2Fdoc%2Fuser_manual.md%23cre
> > ati
> > on-of-local-apt-repo-caching-upstream-debian-
> > packages&data=05%7C02%7Cs
> > rinuvasan.a%40siemens.com%7C9492cda997b94dc4e81708dd5e759d85%7C38ae
> > 3bc
> > d95794fd4addab42e1495d55a%7C1%7C0%7C638770581452457810%7CUnknown%7C
> > TWF
> > pbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMi
> > IsI
> > kFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=3XId9a7IuBK9xyzY
> > Wnu
> > dmeG0aKe3awlNSGSmwId7D%2BU%3D&reserved=0
> >  ):
> > 
> >                 Trigger the warmup build to download deb and deb-
> > src 
> > packages ( Enable the BASE_REPO_FEATURES = "cache-deb-src" in
> > conf/local.conf)
> > 
> >                 e.x: bitbake mc:qemuamd64-bookworm:isar-image-base
> > 
> >                 Once all the deb and deb-src downloaded for the 
> > particular BSP under /download folder
> > 
> >                 Trigger the base-apt target to cache all the deb
> > and 
> > deb-src (Enable ISAR_USE_CACHED_BASE_REPO = "1" in conf/local.conf)
> > 
> >                 e.x: bitbake base-apt
> > 
> >                 You will see the failures:
> >                 
> >                 Logs: 
> >                 
> >                 NOTE: Resolving any missing task queue dependencies
> > Sstate summary: Wanted 0 Local 0 Mirrors 0 Missed 0 Current 0 (0% 
> > match, 0%
> > complete)#############################################            
> > > 
> > ETA:  0:00:00
> > Initialising tasks: 100%
> > > #################################################################
> > > ###
> > ################################################| Time: 0:00:00
> > NOTE: No setscene tasks
> > NOTE: Executing Tasks
> > ERROR: base-apt-1.0-r0 do_cache: One or more sources are missing in
> > repo. bookworm adduser 3.134
> > pool/main/a/adduser/adduser_3.134_all.deb
> > bookworm bmap-tools 3.6-2 pool/main/b/bmap-tools/bmap-tools_3.6-
> > 2_all.deb
> > bookworm debconf 1.5.82 pool/main/d/debconf/debconf_1.5.82_all.deb
> > bookworm debian-archive-keyring 2023.3+deb12u1 pool/main/d/debian- 
> > archive-keyring/debian-archive-keyring_2023.3+deb12u1_all.deb
> > bookworm audit 1:3.0.9-1 pool/main/a/audit/libaudit-common_3.0.9-
> > 1_all.deb
> > bookworm libfile-find-rule-perl 0.34-3 pool/main/libf/libfile-find-
> > rule-perl/libfile-find-rule-perl_0.34-3_all.deb
> > bookworm libnumber-compare-perl 0.03-3 pool/main/libn/libnumber- 
> > compare-perl/libnumber-compare-perl_0.03-3_all.deb
> > bookworm pam 1.5.2-6+deb12u1 pool/main/p/pam/libpam-runtime_1.5.2-
> > 6+deb12u1_all.deb
> 
> .dsc for pam packages found in our downstream project
> 
> Yes cedric, validated from my end, it is available in downstream
> project because we have your changes in isar-patches
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr%2F&data=05%7C02%7Csrinuvasan.a%40siemens.com%7Ccd17f90bd20b4d46a1ff08dd5fb403e6%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771948975584889%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=20glh8vYj2bN2dgIRh6QytScdg1ZjbZzgd9xu6vu18s%3D&reserved=0
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307601863%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=OoUVTbJLnDaPdXKYmZmOh0Wxn9HpiRis%2FYRaL
> YL2Khs%3D&reserved=0)
> 
> But still we can reproduce the issue in vanilla ISAR without your
> patch, I started working on these changes before you proposed these
> changes
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr%2F&data=05%7C02%7Csrinuvasan.a%40siemens.com%7Ccd17f90bd20b4d46a1ff08dd5fb403e6%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771948975596894%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=RvVRIn9snNRs%2BJJjNsWXAcBrkZcuygNcl28CeWpV4dc%3D&reserved=0
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307616236%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=IknRwnuSOZiNUPnx43d4W4dJd2vmylKoUyp%2Bp
> T4W6OE%3D&reserved=0)
> 
> Basically, your patches remove the is_not_part_of_current_build()
> function
> And refactor the debsrc_download functionality.
> 
>  My patch becomes obsoleted if your
> (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgr%2F&data=05%7C02%7Csrinuvasan.a%40siemens.com%7Ccd17f90bd20b4d46a1ff08dd5fb403e6%7C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C638771948975609865%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=34iH9h7A%2BEeaokZRn1B8UKOm9iOS2WQlYCuBCyIjgTA%3D&reserved=0
> oups.google.com%2Fg%2Fisar-
> users%2Fc%2FOwrSBmCPe0Y&data=05%7C02%7Ccedric.hombourger%40siemens.co
> m%7C42bcccafcd8f42e5f6d308dd5ee6f2b3%7C38ae3bcd95794fd4addab42e1495d5
> 5a%7C1%7C0%7C638771068307630537%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hc
> GkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj
> oyfQ%3D%3D%7C0%7C%7C%7C&sdata=vkjpy98ZFZZo4WgKjtwj7UREcd%2B2J4EBcWliJ
> eIdHbU%3D&reserved=0) changes are going to merge in ISAR.
> 
> Please provide your thoughts?


Well the future of my patch and yours are more for the Isar maintainers
to decide. They definitely conflict with each other.

Now the conflicts are resolved and make the solution from your patch and mine, Please refer this PR: https://groups.google.com/g/isar-users/c/fhinJ7Z6Ro4

Many thanks,
Srinu

> 
> > bookworm libsemanage 3.4-1 pool/main/libs/libsemanage/libsemanage-
> > common_3.4-1_all.deb
> > bookworm libtext-glob-perl 0.11-3 pool/main/libt/libtext-glob- 
> > perl/libtext-glob-perl_0.11-3_all.deb
> > bookworm perl 5.36.0-7+deb12u1 pool/main/p/perl/perl-modules- 
> > 5.36_5.36.0-7+deb12u1_all.deb bookworm six 1.16.0-4 
> > pool/main/s/six/python3-six_1.16.0-4_all.deb
> > bookworm tzdata 2024b-0+deb12u1 pool/main/t/tzdata/tzdata_2024b-
> > 0+deb12u1_all.deb
> > bookworm usrmerge 37~deb12u1
> > pool/main/u/usrmerge/usrmerge_37~deb12u1_all.deb
> > bookworm adduser 3.134 pool/main/a/adduser/adduser_3.134_all.deb
> > 
> > Just i captured few packages, but many of the deb packages don't
> > have 
> > their source files
> > 
> > In the base-apt recipes we are calling the repo_sanity_test
> > function 
> > finally to make sure all the deb's having the corresponding deb-src
> > packages should be present in the base-apt.
> 
> I checked again log.do_cache in our downstream project and I did not
> see any of the mentioned errors.
> 
> In downstream project we are not at all calling the repo_sanity_test
> function due to enabled the cache-deb-src in
> ROOTFS_POSTPROCESS_COMMAND rather than enabled in conf/local.conf
> 
> The below condition fails:
> if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src',
> 'yes', 'no', d)}" = "yes" ]
> 
> that is the reason we are not calling this statement at all "
> reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing "
> 
> because we are not enabled the cache-deb-src via BASE_REPO_FEATURES
> 
> 
> $ grep -v 'Export' ./tmp/work/debian-bookworm-any/base-apt/1.0-
> r0/temp/log.do_cache
> DEBUG: Executing python function do_cache
> DEBUG: Using absolute /build/../repo/gnupg/205D601E.pub
> DEBUG: Executing shell function repo
> DEBUG: Shell function repo finished
> DEBUG: Python function do_cache finished
> 
> That build was however done with the patch I have submitted earlier
> to this list. It also includes our own layer. I will do a plain Isar
> build (with my patch) to check if the issue is reproducible.
> 
> I took a few error cases from your example above and verified that
> the .dsc files are present in the generated base-apt build.
> 
> It should be noted that my patch removes
> "is_not_part_of_current_build()"
> 
> parsing dpkg.log was IMO not a great idea as I don't think its format
> is officially documented (I could be wrong though)
> 
> > 
> >                 
> > Yes, as you said, we are parsing deb files (from
> > /var/cache/apt/archives) that were downloaded to determine which 
> > source packages should be downloaded.
> > but in the function "is_not_part_of_current_build "${package}" we
> > are 
> > skipping the download-src if parsing deb package (status installed)
> > is 
> > not available in the below files ```
> >             "${IMAGE_ROOTFS}"/var/log/dpkg.log \
> >             "${SCHROOT_HOST_DIR}"/var/log/dpkg.log \
> >             "${SCHROOT_TARGET_DIR}"/var/log/dpkg.log \
> >             "${SCHROOT_HOST_DIR}"/tmp/dpkg_common.log \
> >             "${SCHROOT_TARGET_DIR}"/tmp/dpkg_common.log
> > ```
> > 
> > As I mentioned above, the adduser package was installed in the
> > stage 
> > of bootstrap, but that package's status was not captured in the 
> > bootstrap's /var/log/dpkg.log file.
> > because mmdebstrap deletes those files by default, hence we didn't 
> > capture those bootstrap package status.
> > 
> > As we know bootstrap rootfs is used for sbuildchroot creation, but 
> > here whatever packages are installed during sbuild-chroot creation,
> > ${SCHROOT_TARGET_DIR}"/var/log/dpkg.log this file only contains the
> > sbuildchroot related package status, but our expectation is it
> > should 
> > have bootstrap + sbuildchroot package status available in 
> > /var/log/dpkg.log file in sbuildchroot.
> > 
> > My first patch solves the above issue
> > with the introduction of "skip=cleanup/reproducible", the bootstrap
> > rootfs generated with /var/log/dpkg.log files, which contain all
> > the 
> > bootstrap package status along with that if you use that boostrap
> > for 
> > sbuildchroot creation , the sbuildchroot rootfs contains the
> > existing 
> > bootstrap plus additionally installed sbuildchroot related packages
> > status is available in sbuldchroot /var/log/dpkg.log file
> > 
> > 
> > My second patch resolve the below issue:
> > 
> > During image creation (IMAGE_FSTYPES = "wic") few packages were 
> > installed in the session (e.g: gdisk, bmap-tools) but those
> > packages 
> > list available only in the sbuild environment, not outside, carry 
> > those packages status to tmp/dpkg_common.log in the rootfs to get
> > the 
> > src packages for imager_install related deb packages, presently the
> > cache-deb-src function calls in the ROOTFS_POSTPROCESS_COMMAND but
> > we 
> > need to wait till imager generation That's why we pushed that 
> > deb-src-caching after image creation.
> > 
> > My third path resolve the image rootfs related deb-src missing
> > issue:
> > 
> > By default we are removing the /var/log/dpkg.log file in the 
> > rootfs_postprocess_clean_log_files function, but this is needed
> > when 
> > We enable the BASE_REPO_FEATURES = "cache-deb-src", hence making
> > this 
> > function as a configurable one rather than enabled by default.
> > to download IMAGE_ROOTFS related deb-src packages
> > 
> > I hope I explain the issue, presently am checking the reproducible 
> > part (for jan queries), will update that part very soon.
> > 
> > Many thanks,
> > Srinu
> > 
> > > > 
> > > > Signed-off-by: srinuvasan <srinuvasan.a@siemens.com>
> > > > ---
> > > >  meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 6
> > > > ++++++
> > > >  1 file changed, 6 insertions(+)
> > > > 
> > > > diff --git a/meta/recipes-core/isar-mmdebstrap/isar-
> > > > mmdebstrap.inc
> > > > b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > index 931f6f13..7380a658 100644
> > > > --- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> > > > @@ -168,6 +168,11 @@ do_bootstrap() {
> > > >          arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> > > >      fi
> > > >  
> > > > +    cleanup=
> > > > +    if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-
> > > > deb-
> > > > src', 'yes', 'no', d)}" = "yes" ]; then
> > > > +        cleanup="--skip=cleanup/reproducible"
> > > > +    fi
> > > > +
> > > >      # Cleanup mounts if fails
> > > >      trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> > > >      trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat
> > > > "${WORKDIR}/mmtmpdir") \
> > > > @@ -183,6 +188,7 @@ do_bootstrap() {
> > > >  
> > > >      sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap
> > > > $bootstrap_args 
> > > > \
> > > >                     $arch_param \
> > > > +                   $cleanup \
> > > >                     --mode=unshare \
> > > >                     ${MMHOOKS} \
> > > >                     --setup-hook='mkdir -p 
> > > > "$1/var/cache/apt/archives/"' \
> > > 
> > > What has changed in this version? I'm not seeing my concerns 
> > > addressed, my questions answered.
> > 
> > +1
> > 
> > Please make sure new revisions of your patch series include a 
> > changelog (use --cover-letter with git format-patch)
> > 
> > Sure
> > 
> > > 
> > > Jan
> > > 
> > 
> > 
> 
> --
> Cedric Hombourger
> Siemens AG
> www.siemens.com

-- 
Cedric Hombourger
Siemens AG
www.siemens.com

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/PUZPR06MB5747B46FF7ADCDF721DBC4BC88D62%40PUZPR06MB5747.apcprd06.prod.outlook.com.