[PATCH v7] suggested changes for reproducibility patchset v7

[PATCH v7] suggested changes for reproducibility patchset v7

[roberto.foglietta]

From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>

suggested changes for reproducibility patchset

WARNING: eval-image-1.0-r0 do_rootfs_finalize: modified timestamp (1673628837) of 3 files for image reproducibly
         List of files modified could be found here: ./build/tmp/deploy/images/debx86/files.modified_timestamps

v.2: rebased on current ilbers:next

v.3: new script added: wic-extract-rootfs-partition.sh [image.wic]

v.4: example with for epoch generation from git

v.5: reverted the example and rework some few code

v.6: the 1st part of the warning shows up each time the epoch is used
     while the 2nd line appears only when some files has been touched
     This allows the user to know the current situation aboat epoch.

v.7: forgot to commit before producing the patch v6 but sent!

Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>

produc

Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
---
 meta-isar/conf/local.conf.sample             |  2 +-
 meta/classes/image-account-extension.bbclass |  6 +--
 meta/classes/image.bbclass                   | 22 +++++----
 meta/classes/initramfs.bbclass               |  4 +-
 wic-extract-rootfs-partition.sh              | 52 ++++++++++++++++++++
 5 files changed, 71 insertions(+), 15 deletions(-)
 create mode 100755 wic-extract-rootfs-partition.sh

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 6208623e..1d7e178a 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -257,4 +257,4 @@ USER_isar[flags] += "clear-text-password"
 # Non git repository users can use value from 'stat -c%Y ChangeLog'
 # To know more details about this variable and how to set the value refer below
 # https://reproducible-builds.org/docs/source-date-epoch/
-#SOURCE_DATE_EPOCH =
+#SOURCE_DATE_EPOCH = ""
diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
index bb173b14..1d49054c 100644
--- a/meta/classes/image-account-extension.bbclass
+++ b/meta/classes/image-account-extension.bbclass
@@ -256,11 +256,11 @@ image_postprocess_accounts() {
                 # chpasswd adds a random salt when running against a clear-text password.
                 # For reproducible images, we manually generate the password and use the
                 # SOURCE_DATE_EPOCH to generate the salt in a deterministic way.
-                if [ -z "${SOURCE_DATE_EPOCH}"]; then
+                if [ -z "${SOURCE_DATE_EPOCH}" ]; then
                     chpasswd_args=""
                 else
-                    salt="$(echo "${SOURCE_DATE_EPOCH}" | sha256sum -z | cut -c 1-15)"
-                    password="$(openssl passwd -6 -salt $salt "$password")"
+                    salt="$(echo ${SOURCE_DATE_EPOCH} | sha256sum -z | cut -c 1-15)"
+                    password="$(openssl passwd -6 -salt $salt $password)"
                 fi
             fi
             printf '%s:%s' "$name" "$password" | sudo chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 063b9a3b..191c3940 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -310,8 +310,8 @@ python() {
 #       invalidate the SSTATE entries for most packages, even if they don't use the
 #       global SOURCE_DATE_EPOCH variable.
 rootfs_install_pkgs_install_prepend() {
-    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
-        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
+        export SOURCE_DATE_EPOCH
     fi
 }
 
@@ -443,13 +443,17 @@ EOSUDO
 
     # Set same time-stamps to the newly generated file/folders in the
     # rootfs image for the purpose of reproducible builds.
-    test ! -z "${SOURCE_DATE_EPOCH}" && \
-        sudo find ${ROOTFSDIR} -newermt \
-            "$(date -d@${SOURCE_DATE_EPOCH} '+%Y-%m-%d %H:%M:%S')" \
-            -printf "%y %p\n" \
-            -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' > ${DEPLOY_DIR_IMAGE}/files.modified_timestamps && \
-            bbwarn "$(grep ^f ${DEPLOY_DIR_IMAGE}/files.modified_timestamps) \nModified above file timestamps to build image reproducibly"
-
+    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
+        fn="${DEPLOY_DIR_IMAGE}/files.modified_timestamps"
+        sudo find ${ROOTFSDIR} -newermt "$(date -d@"${SOURCE_DATE_EPOCH}" '+%Y-%m-%d %H:%M:%S')" \
+            -printf "%y %p\n" -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' >"$fn"
+        msg=""
+        ncfs=$(egrep ^f "$fn" | wc -l)
+        if [ $ncfs -gt 0 ]; then
+            msg="\n         List of files modified could be found here: ."${DEPLOY_DIR_IMAGE}"/files.modified_timestamps"
+        fi
+        bbwarn "Modified timestamp ("${SOURCE_DATE_EPOCH}") of "$ncfs" files for image reproducibly.$msg"
+    fi
 }
 addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
 
diff --git a/meta/classes/initramfs.bbclass b/meta/classes/initramfs.bbclass
index db283347..1b98bc06 100644
--- a/meta/classes/initramfs.bbclass
+++ b/meta/classes/initramfs.bbclass
@@ -33,8 +33,8 @@ do_generate_initramfs() {
     rootfs_do_qemu
 
     # generate reproducible initrd if requested
-    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
-        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
+        export SOURCE_DATE_EPOCH
     fi
 
     sudo -E chroot "${INITRAMFS_ROOTFS}" \
diff --git a/wic-extract-rootfs-partition.sh b/wic-extract-rootfs-partition.sh
new file mode 100755
index 00000000..48de0d3a
--- /dev/null
+++ b/wic-extract-rootfs-partition.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright (c) Roberto A. Foglietta, 2023
+#
+# Authors:
+#  Roberto A. Foglietta <roberto.foglietta@gmail.com>
+#
+# SPDX-License-Identifier: MIT
+#
+#set -ex
+
+if [ "$(whoami)" != "root" ]; then
+    echo
+    echo "WARNING: this script should run as root, sudo!"
+    sudo -E $0 "$@"
+    exit $?
+fi
+
+if [ -e "$1" ]; then
+    fimg=$(readlink -e $1)
+fi
+
+cd $(dirname $0)
+
+if [ ! -n "$1" -a  ! -e "$fimg" ]; then
+    fimg=$(ls -1 build/tmp/deploy/images/*/*.wic)
+    n=( $fimg )
+    if [ ${#n[@]} -gt 1 ]; then
+        echo
+        echo "WARNING: more than one image found, choose one:"
+        echo
+        echo "$fimg"
+        echo
+        exit 1
+    fi
+fi
+
+if [ ! -e "$fimg" ]; then
+    echo
+    echo "ERROR: no any image or block device ${1:+'$1' }found, abort!"
+    echo
+    exit 1
+fi
+
+wicf=$fimg
+losetup -Pf $wicf
+ldev=$(losetup -j $wicf | cut -d: -f1 | tail -n1)
+echo loopdev:$ldev
+dd if=${ldev}p2 bs=1M of=${wicf/.wic/.rootfs} status=progress
+chown $(id -u).$(id -g) ${wicf/.wic/.rootfs}
+du -ms ${wicf/.wic/.rootfs}
+losetup -d $ldev
-- 
2.34.1

Re: [PATCH v7] suggested changes for reproducibility patchset v7

[Florian Bezdeka]

On Sun, 2023-01-15 at 23:17 +0100, roberto.foglietta@linuxteam.org
wrote:
> From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>
> 
> suggested changes for reproducibility patchset
> 
> WARNING: eval-image-1.0-r0 do_rootfs_finalize: modified timestamp (1673628837) of 3 files for image reproducibly
>          List of files modified could be found here: ./build/tmp/deploy/images/debx86/files.modified_timestamps
> 
> v.2: rebased on current ilbers:next
> 
> v.3: new script added: wic-extract-rootfs-partition.sh [image.wic]
> 
> v.4: example with for epoch generation from git
> 
> v.5: reverted the example and rework some few code
> 
> v.6: the 1st part of the warning shows up each time the epoch is used
>      while the 2nd line appears only when some files has been touched
>      This allows the user to know the current situation aboat epoch.
> 
> v.7: forgot to commit before producing the patch v6 but sent!
> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> 
> produc
^^^^^^^^^ What? Please also note the comments made to v6. Overlooked v7
in my first round some minutes ago.

> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>

Duplicate Signed-off-by.

> ---
>  meta-isar/conf/local.conf.sample             |  2 +-
>  meta/classes/image-account-extension.bbclass |  6 +--
>  meta/classes/image.bbclass                   | 22 +++++----
>  meta/classes/initramfs.bbclass               |  4 +-
>  wic-extract-rootfs-partition.sh              | 52 ++++++++++++++++++++
>  5 files changed, 71 insertions(+), 15 deletions(-)
>  create mode 100755 wic-extract-rootfs-partition.sh
> 
> diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
> index 6208623e..1d7e178a 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -257,4 +257,4 @@ USER_isar[flags] += "clear-text-password"
>  # Non git repository users can use value from 'stat -c%Y ChangeLog'
>  # To know more details about this variable and how to set the value refer below
>  # https://reproducible-builds.org/docs/source-date-epoch/
> -#SOURCE_DATE_EPOCH =
> +#SOURCE_DATE_EPOCH = ""
> diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
> index bb173b14..1d49054c 100644
> --- a/meta/classes/image-account-extension.bbclass
> +++ b/meta/classes/image-account-extension.bbclass
> @@ -256,11 +256,11 @@ image_postprocess_accounts() {
>                  # chpasswd adds a random salt when running against a clear-text password.
>                  # For reproducible images, we manually generate the password and use the
>                  # SOURCE_DATE_EPOCH to generate the salt in a deterministic way.
> -                if [ -z "${SOURCE_DATE_EPOCH}"]; then
> +                if [ -z "${SOURCE_DATE_EPOCH}" ]; then
>                      chpasswd_args=""
>                  else
> -                    salt="$(echo "${SOURCE_DATE_EPOCH}" | sha256sum -z | cut -c 1-15)"
> -                    password="$(openssl passwd -6 -salt $salt "$password")"
> +                    salt="$(echo ${SOURCE_DATE_EPOCH} | sha256sum -z | cut -c 1-15)"
> +                    password="$(openssl passwd -6 -salt $salt $password)"
>                  fi
>              fi
>              printf '%s:%s' "$name" "$password" | sudo chroot '${ROOTFSDIR}' \
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index 063b9a3b..191c3940 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -310,8 +310,8 @@ python() {
>  #       invalidate the SSTATE entries for most packages, even if they don't use the
>  #       global SOURCE_DATE_EPOCH variable.
>  rootfs_install_pkgs_install_prepend() {
> -    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> -        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        export SOURCE_DATE_EPOCH
>      fi
>  }
>  
> @@ -443,13 +443,17 @@ EOSUDO
>  
>      # Set same time-stamps to the newly generated file/folders in the
>      # rootfs image for the purpose of reproducible builds.
> -    test ! -z "${SOURCE_DATE_EPOCH}" && \
> -        sudo find ${ROOTFSDIR} -newermt \
> -            "$(date -d@${SOURCE_DATE_EPOCH} '+%Y-%m-%d %H:%M:%S')" \
> -            -printf "%y %p\n" \
> -            -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' > ${DEPLOY_DIR_IMAGE}/files.modified_timestamps && \
> -            bbwarn "$(grep ^f ${DEPLOY_DIR_IMAGE}/files.modified_timestamps) \nModified above file timestamps to build image reproducibly"
> -
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        fn="${DEPLOY_DIR_IMAGE}/files.modified_timestamps"
> +        sudo find ${ROOTFSDIR} -newermt "$(date -d@"${SOURCE_DATE_EPOCH}" '+%Y-%m-%d %H:%M:%S')" \
> +            -printf "%y %p\n" -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' >"$fn"
> +        msg=""
> +        ncfs=$(egrep ^f "$fn" | wc -l)
> +        if [ $ncfs -gt 0 ]; then
> +            msg="\n         List of files modified could be found here: ."${DEPLOY_DIR_IMAGE}"/files.modified_timestamps"
> +        fi
> +        bbwarn "Modified timestamp ("${SOURCE_DATE_EPOCH}") of "$ncfs" files for image reproducibly.$msg"
> +    fi
>  }
>  addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
>  
> diff --git a/meta/classes/initramfs.bbclass b/meta/classes/initramfs.bbclass
> index db283347..1b98bc06 100644
> --- a/meta/classes/initramfs.bbclass
> +++ b/meta/classes/initramfs.bbclass
> @@ -33,8 +33,8 @@ do_generate_initramfs() {
>      rootfs_do_qemu
>  
>      # generate reproducible initrd if requested
> -    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> -        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        export SOURCE_DATE_EPOCH
>      fi
>  
>      sudo -E chroot "${INITRAMFS_ROOTFS}" \
> diff --git a/wic-extract-rootfs-partition.sh b/wic-extract-rootfs-partition.sh
> new file mode 100755
> index 00000000..48de0d3a
> --- /dev/null
> +++ b/wic-extract-rootfs-partition.sh
> @@ -0,0 +1,52 @@
> +#!/bin/bash
> +#
> +# Copyright (c) Roberto A. Foglietta, 2023
> +#
> +# Authors:
> +#  Roberto A. Foglietta <roberto.foglietta@gmail.com>
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +#set -ex
> +
> +if [ "$(whoami)" != "root" ]; then
> +    echo
> +    echo "WARNING: this script should run as root, sudo!"
> +    sudo -E $0 "$@"
> +    exit $?
> +fi
> +
> +if [ -e "$1" ]; then
> +    fimg=$(readlink -e $1)
> +fi
> +
> +cd $(dirname $0)
> +
> +if [ ! -n "$1" -a  ! -e "$fimg" ]; then
> +    fimg=$(ls -1 build/tmp/deploy/images/*/*.wic)
> +    n=( $fimg )
> +    if [ ${#n[@]} -gt 1 ]; then
> +        echo
> +        echo "WARNING: more than one image found, choose one:"
> +        echo
> +        echo "$fimg"
> +        echo
> +        exit 1
> +    fi
> +fi
> +
> +if [ ! -e "$fimg" ]; then
> +    echo
> +    echo "ERROR: no any image or block device ${1:+'$1' }found, abort!"
> +    echo
> +    exit 1
> +fi
> +
> +wicf=$fimg
> +losetup -Pf $wicf
> +ldev=$(losetup -j $wicf | cut -d: -f1 | tail -n1)
> +echo loopdev:$ldev
> +dd if=${ldev}p2 bs=1M of=${wicf/.wic/.rootfs} status=progress
> +chown $(id -u).$(id -g) ${wicf/.wic/.rootfs}
> +du -ms ${wicf/.wic/.rootfs}
> +losetup -d $ldev
> -- 
> 2.34.1
> 

Re: [PATCH v7] suggested changes for reproducibility patchset v7

[Roberto A. Foglietta]

[-- Attachment #1: Type: text/plain, Size: 642 bytes --]

On Sun, 15 Jan 2023 at 23:33, Florian Bezdeka <florian.bezdeka@siemens.com>
wrote:

> >
> > v.7: forgot to commit before producing the patch v6 but sent!
> >
> > Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> >
> > produc
> ^^^^^^^^^ What? Please also note the comments made to v6. Overlooked v7
> in my first round some minutes ago.


Your message about this suggestion was 15 later the submit of the patch v7,
so I did not read yet at the time of this submission.

Moreover: do not worry, I will do a proper patch when your changes will be
included into ilbers/next - this is just a suggestion for Felix

Best regards,

[-- Attachment #2: Type: text/html, Size: 1154 bytes --]