Re: [PATCH v7] suggested changes for reproducibility patchset v7

[Florian Bezdeka <florian.bezdeka@siemens.com>]

On Sun, 2023-01-15 at 23:17 +0100, roberto.foglietta@linuxteam.org
wrote:
> From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>
> 
> suggested changes for reproducibility patchset
> 
> WARNING: eval-image-1.0-r0 do_rootfs_finalize: modified timestamp (1673628837) of 3 files for image reproducibly
>          List of files modified could be found here: ./build/tmp/deploy/images/debx86/files.modified_timestamps
> 
> v.2: rebased on current ilbers:next
> 
> v.3: new script added: wic-extract-rootfs-partition.sh [image.wic]
> 
> v.4: example with for epoch generation from git
> 
> v.5: reverted the example and rework some few code
> 
> v.6: the 1st part of the warning shows up each time the epoch is used
>      while the 2nd line appears only when some files has been touched
>      This allows the user to know the current situation aboat epoch.
> 
> v.7: forgot to commit before producing the patch v6 but sent!
> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> 
> produc
^^^^^^^^^ What? Please also note the comments made to v6. Overlooked v7
in my first round some minutes ago.

> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>

Duplicate Signed-off-by.

> ---
>  meta-isar/conf/local.conf.sample             |  2 +-
>  meta/classes/image-account-extension.bbclass |  6 +--
>  meta/classes/image.bbclass                   | 22 +++++----
>  meta/classes/initramfs.bbclass               |  4 +-
>  wic-extract-rootfs-partition.sh              | 52 ++++++++++++++++++++
>  5 files changed, 71 insertions(+), 15 deletions(-)
>  create mode 100755 wic-extract-rootfs-partition.sh
> 
> diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
> index 6208623e..1d7e178a 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -257,4 +257,4 @@ USER_isar[flags] += "clear-text-password"
>  # Non git repository users can use value from 'stat -c%Y ChangeLog'
>  # To know more details about this variable and how to set the value refer below
>  # https://reproducible-builds.org/docs/source-date-epoch/
> -#SOURCE_DATE_EPOCH =
> +#SOURCE_DATE_EPOCH = ""
> diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
> index bb173b14..1d49054c 100644
> --- a/meta/classes/image-account-extension.bbclass
> +++ b/meta/classes/image-account-extension.bbclass
> @@ -256,11 +256,11 @@ image_postprocess_accounts() {
>                  # chpasswd adds a random salt when running against a clear-text password.
>                  # For reproducible images, we manually generate the password and use the
>                  # SOURCE_DATE_EPOCH to generate the salt in a deterministic way.
> -                if [ -z "${SOURCE_DATE_EPOCH}"]; then
> +                if [ -z "${SOURCE_DATE_EPOCH}" ]; then
>                      chpasswd_args=""
>                  else
> -                    salt="$(echo "${SOURCE_DATE_EPOCH}" | sha256sum -z | cut -c 1-15)"
> -                    password="$(openssl passwd -6 -salt $salt "$password")"
> +                    salt="$(echo ${SOURCE_DATE_EPOCH} | sha256sum -z | cut -c 1-15)"
> +                    password="$(openssl passwd -6 -salt $salt $password)"
>                  fi
>              fi
>              printf '%s:%s' "$name" "$password" | sudo chroot '${ROOTFSDIR}' \
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index 063b9a3b..191c3940 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -310,8 +310,8 @@ python() {
>  #       invalidate the SSTATE entries for most packages, even if they don't use the
>  #       global SOURCE_DATE_EPOCH variable.
>  rootfs_install_pkgs_install_prepend() {
> -    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> -        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        export SOURCE_DATE_EPOCH
>      fi
>  }
>  
> @@ -443,13 +443,17 @@ EOSUDO
>  
>      # Set same time-stamps to the newly generated file/folders in the
>      # rootfs image for the purpose of reproducible builds.
> -    test ! -z "${SOURCE_DATE_EPOCH}" && \
> -        sudo find ${ROOTFSDIR} -newermt \
> -            "$(date -d@${SOURCE_DATE_EPOCH} '+%Y-%m-%d %H:%M:%S')" \
> -            -printf "%y %p\n" \
> -            -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' > ${DEPLOY_DIR_IMAGE}/files.modified_timestamps && \
> -            bbwarn "$(grep ^f ${DEPLOY_DIR_IMAGE}/files.modified_timestamps) \nModified above file timestamps to build image reproducibly"
> -
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        fn="${DEPLOY_DIR_IMAGE}/files.modified_timestamps"
> +        sudo find ${ROOTFSDIR} -newermt "$(date -d@"${SOURCE_DATE_EPOCH}" '+%Y-%m-%d %H:%M:%S')" \
> +            -printf "%y %p\n" -exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';' >"$fn"
> +        msg=""
> +        ncfs=$(egrep ^f "$fn" | wc -l)
> +        if [ $ncfs -gt 0 ]; then
> +            msg="\n         List of files modified could be found here: ."${DEPLOY_DIR_IMAGE}"/files.modified_timestamps"
> +        fi
> +        bbwarn "Modified timestamp ("${SOURCE_DATE_EPOCH}") of "$ncfs" files for image reproducibly.$msg"
> +    fi
>  }
>  addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
>  
> diff --git a/meta/classes/initramfs.bbclass b/meta/classes/initramfs.bbclass
> index db283347..1b98bc06 100644
> --- a/meta/classes/initramfs.bbclass
> +++ b/meta/classes/initramfs.bbclass
> @@ -33,8 +33,8 @@ do_generate_initramfs() {
>      rootfs_do_qemu
>  
>      # generate reproducible initrd if requested
> -    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> -        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> +    if [ -n "${SOURCE_DATE_EPOCH}" ]; then
> +        export SOURCE_DATE_EPOCH
>      fi
>  
>      sudo -E chroot "${INITRAMFS_ROOTFS}" \
> diff --git a/wic-extract-rootfs-partition.sh b/wic-extract-rootfs-partition.sh
> new file mode 100755
> index 00000000..48de0d3a
> --- /dev/null
> +++ b/wic-extract-rootfs-partition.sh
> @@ -0,0 +1,52 @@
> +#!/bin/bash
> +#
> +# Copyright (c) Roberto A. Foglietta, 2023
> +#
> +# Authors:
> +#  Roberto A. Foglietta <roberto.foglietta@gmail.com>
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +#set -ex
> +
> +if [ "$(whoami)" != "root" ]; then
> +    echo
> +    echo "WARNING: this script should run as root, sudo!"
> +    sudo -E $0 "$@"
> +    exit $?
> +fi
> +
> +if [ -e "$1" ]; then
> +    fimg=$(readlink -e $1)
> +fi
> +
> +cd $(dirname $0)
> +
> +if [ ! -n "$1" -a  ! -e "$fimg" ]; then
> +    fimg=$(ls -1 build/tmp/deploy/images/*/*.wic)
> +    n=( $fimg )
> +    if [ ${#n[@]} -gt 1 ]; then
> +        echo
> +        echo "WARNING: more than one image found, choose one:"
> +        echo
> +        echo "$fimg"
> +        echo
> +        exit 1
> +    fi
> +fi
> +
> +if [ ! -e "$fimg" ]; then
> +    echo
> +    echo "ERROR: no any image or block device ${1:+'$1' }found, abort!"
> +    echo
> +    exit 1
> +fi
> +
> +wicf=$fimg
> +losetup -Pf $wicf
> +ldev=$(losetup -j $wicf | cut -d: -f1 | tail -n1)
> +echo loopdev:$ldev
> +dd if=${ldev}p2 bs=1M of=${wicf/.wic/.rootfs} status=progress
> +chown $(id -u).$(id -g) ${wicf/.wic/.rootfs}
> +du -ms ${wicf/.wic/.rootfs}
> +losetup -d $ldev
> -- 
> 2.34.1
>