Re: [PATCH 1/1] Reduce disk usage during build by avoiding copies of apt cache

[Henning Schild <henning.schild@siemens.com>]

Am Wed,  4 Jan 2023 02:31:28 +0000
schrieb Felix Moessbauer <felix.moessbauer@siemens.com>:

> From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>
> 
> This patch leverages few ways to accomplish the task to optimise the
> large buildings. 

like the empire state building?

I find this patch hard to review, it seems to do multiple things at
once. Things that others have understood already so i will not review.

But if it needs to be send again, maybe we can split it up. If any of
that causes issues it will be hard to blame and revert. And the authors
might not understand it anymore in ... say a year or less.

Henning

> It uses hard physical links instead of copying files
> which is a faster way and saves disk spaces. Put a CACHEDIR.TAG file
> in each cache folder in such a way every tar that uses the option
> --exclude-caches could be avoided to include debian packages saving a
> lot of time and disk space.
> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
> 
> This is a cleaned up and squashed version of the following commits
> from Robertos ISAR fork:
> 
> - 83ce56580c39a2f708f9a3ede73439c82edac1be
> - 02bfafb20a8c43585561b9de306a2e23c5ff50bb
> 
> In addition, I applied some small fixes to fix issues on rebuild.
> This is a compromise between the large patch series of Roberto and
> the too minimalistic one of Uladzimir. As the series from Roberto is
> not on this list, I want to send out this patch to have something
> which we can review.
> 
> Maybe we can integrate that first to solve 90% of the issue and then
> continue improving it.
> 
> Best regards,
> Felix
> 
>  meta/classes/deb-dl-dir.bbclass                     | 11 ++++++-----
>  meta/classes/dpkg.bbclass                           |  6 +++---
>  meta/classes/rootfs.bbclass                         |  8 ++++----
>  meta/classes/sdk.bbclass                            |  2 +-
>  meta/recipes-core/isar-bootstrap/isar-bootstrap.inc |  3 ++-
>  5 files changed, 16 insertions(+), 14 deletions(-)
> 
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index 3b1517d..3a87390 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -84,9 +84,9 @@ deb_dl_dir_import() {
>      flock -s "${pc}".lock -c '
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> -
> +        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
>          sudo find "${pc}" -type f -iname "*\.deb" -exec \
> -            cp -n --no-preserve=owner -t
> "${rootfs}"/var/cache/apt/archives/ {} +
> +            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ {} +
> 2>/dev/null || : '
>  }
>  
> @@ -97,20 +97,21 @@ deb_dl_dir_export() {
>      flock "${pc}".lock -c '
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> -
> +        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
>          find "${rootfs}"/var/cache/apt/archives/ \
>              -maxdepth 1 -type f -iname '*\.deb' |\
>          while read p; do
>              # skip files from a previous export
> -            [ -f "${pc}/${p##*/}" ] && continue
> +            [ -e "${pc}/${p##*/}" ] && continue
>              # can not reuse bitbake function here, this is basically
>              # "repo_contains_package"
>              package=$(find "${REPO_ISAR_DIR}"/"${DISTRO}" -name
> ${p##*/}) if [ -n "$package" ]; then
>                  cmp --silent "$package" "$p" && continue
>              fi
> -            sudo cp -n "${p}" "${pc}"
> +            sudo ln -P "${p}" "${pc}" 2>/dev/null || :
>          done
> +        sudo touch "${pc}/CACHEDIR.TAG"
>          sudo chown -R $(id -u):$(id -g) "${pc}"
>      '
>  }
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index 7822b14..166ec16 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -108,10 +108,10 @@ dpkg_runbuild() {
>          --chroot-setup-commands="echo \"Package: *\nPin: release
> n=${DEBDISTRONAME}\nPin-Priority: 1000\" >
> /etc/apt/preferences.d/isar-apt" \ --chroot-setup-commands="echo
> \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
> --chroot-setup-commands="rm -f /var/log/dpkg.log" \
> -        --chroot-setup-commands="cp -n --no-preserve=owner
> ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \
> +        --chroot-setup-commands="ln -Pf ${ext_deb_dir}/*.deb -t
> ${deb_dir}/ 2>/dev/null || :" \ --finished-build-commands="rm -f
> ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
> -        --finished-build-commands="cp -n --no-preserve=owner
> ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \
> -        --finished-build-commands="cp /var/log/dpkg.log
> ${ext_root}/dpkg_partial.log" \
> +        --finished-build-commands="ln -P ${deb_dir}/*.deb -t
> ${ext_deb_dir}/ 2>/dev/null || :" \
> +        --finished-build-commands="cp /var/log/dpkg.log
> ${ext_root}/dpkg_partial.log; touch ${ext_deb_dir}/CACHEDIR.TAG" \
> --debbuildopts="--source-option=-I" \ --build-dir=${WORKDIR}
> --dist="isar" ${DSC_FILE} 
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 786682d..147f517 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -78,7 +78,7 @@ BOOTSTRAP_SRC_${ROOTFS_ARCH} =
> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTF 
>  rootfs_prepare[weight] = "25"
>  rootfs_prepare(){
> -    sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> +    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>  }
>  
>  ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
> @@ -115,7 +115,6 @@ rootfs_configure_apt() {
>  EOSUDO
>  }
>  
> -
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>  rootfs_install_pkgs_update[weight] = "5"
>  rootfs_install_pkgs_update[isar-apt-lock] = "acquire-before"
> @@ -218,7 +217,7 @@ cache_deb_src() {
>      # Note: ISAR updates the apt state information(apt-get update)
> only once during bootstrap and # relies on that through out the
> build. Copy that state information instead of apt-get update # which
> generates a new state from upstream.
> -    sudo cp -Trpn "${BOOTSTRAP_SRC}/var/lib/apt/lists/"
> "${ROOTFSDIR}/var/lib/apt/lists/"
> +    sudo cp -Trpn --reflink=auto
> "${BOOTSTRAP_SRC}/var/lib/apt/lists/"
> "${ROOTFSDIR}/var/lib/apt/lists/" deb_dl_dir_import ${ROOTFSDIR}
> ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} debsrc_download
> ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME} @@ -308,7
> +307,8 @@ rootfs_install_sstate_prepare() { # so we use some mount
> magic to prevent that mkdir -p ${WORKDIR}/mnt/rootfs
>      sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
> -    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar --one-file-system
> rootfs
> +    lopts="--one-file-system --exclude-caches
> --exclude=var/cache/apt/archives"
> +    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar $lopts rootfs
>      sudo umount ${WORKDIR}/mnt/rootfs
>      sudo chown $(id -u):$(id -g) rootfs.tar
>  }
> diff --git a/meta/classes/sdk.bbclass b/meta/classes/sdk.bbclass
> index 79e01a1..3feada2 100644
> --- a/meta/classes/sdk.bbclass
> +++ b/meta/classes/sdk.bbclass
> @@ -82,7 +82,7 @@ do_rootfs_install[vardeps] += "${SDKROOTFSVARDEPS}"
>  ROOTFS_CONFIGURE_COMMAND_append_class-sdk = "
> ${@'rootfs_configure_isar_apt_dir' if
> d.getVar('SDK_INCLUDE_ISAR_APT') == '1' else ''}"
> rootfs_configure_isar_apt_dir() { # Copy isar-apt instead of mounting:
> -    sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
> +    sudo cp -Trpfx --reflink=auto ${REPO_ISAR_DIR}/${DISTRO}
> ${ROOTFSDIR}/isar-apt }
>  
>  ROOTFS_POSTPROCESS_COMMAND_prepend_class-sdk =
> "sdkchroot_configscript " diff --git
> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> b9ae16c..d53db4d 100644 ---
> a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -418,7
> +418,8 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> bootstrap_sstate_prepare() { # this runs in SSTATE_BUILDDIR, which
> will be deleted automatically
> -    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar
> --one-file-system $(basename "${ROOTFSDIR}")
> +    lopts="--one-file-system --exclude-caches
> --exclude=var/cache/apt/archives"
> +    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts
> $(basename "${ROOTFSDIR}") sudo chown $(id -u):$(id -g) bootstrap.tar
>  }
>