RE: [PATCH 1/1] Reduce disk usage during build by avoiding copies of apt cache

["Schmidt, Adriaan" <adriaan.schmidt@siemens.com>]

Felix Moessbauer, Mittwoch, 4. Januar 2023 03:31:
> Subject: [PATCH 1/1] Reduce disk usage during build by avoiding copies of apt
> cache
> 
> From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>
> 
> This patch leverages few ways to accomplish the task to optimise the large
> buildings. It uses hard physical links instead of copying files which is a
> faster way and saves disk spaces. Put a CACHEDIR.TAG file in each cache
> folder
> in such a way every tar that uses the option --exclude-caches could be
> avoided
> to include debian packages saving a lot of time and disk space.
> 
> Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
> 
> This is a cleaned up and squashed version of the following commits
> from Robertos ISAR fork:
> 
> - 83ce56580c39a2f708f9a3ede73439c82edac1be
> - 02bfafb20a8c43585561b9de306a2e23c5ff50bb
> 
> In addition, I applied some small fixes to fix issues on rebuild.
> This is a compromise between the large patch series of Roberto and
> the too minimalistic one of Uladzimir. As the series from Roberto is
> not on this list, I want to send out this patch to have something which
> we can review.
> 
> Maybe we can integrate that first to solve 90% of the issue and then
> continue improving it.
> 
> Best regards,
> Felix
> 
>  meta/classes/deb-dl-dir.bbclass                     | 11 ++++++-----
>  meta/classes/dpkg.bbclass                           |  6 +++---
>  meta/classes/rootfs.bbclass                         |  8 ++++----
>  meta/classes/sdk.bbclass                            |  2 +-
>  meta/recipes-core/isar-bootstrap/isar-bootstrap.inc |  3 ++-
>  5 files changed, 16 insertions(+), 14 deletions(-)
> 
> diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-
> dir.bbclass
> index 3b1517d..3a87390 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -84,9 +84,9 @@ deb_dl_dir_import() {
>      flock -s "${pc}".lock -c '
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> -
> +        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
>          sudo find "${pc}" -type f -iname "*\.deb" -exec \
> -            cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/
> {} +
> +            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ {} + 2>/dev/null
> || :
>      '
>  }
> 
> @@ -97,20 +97,21 @@ deb_dl_dir_export() {
>      flock "${pc}".lock -c '
>          set -e
>          printenv | grep -q BB_VERBOSE_LOGS && set -x
> -
> +        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
>          find "${rootfs}"/var/cache/apt/archives/ \
>              -maxdepth 1 -type f -iname '*\.deb' |\
>          while read p; do
>              # skip files from a previous export
> -            [ -f "${pc}/${p##*/}" ] && continue
> +            [ -e "${pc}/${p##*/}" ] && continue
>              # can not reuse bitbake function here, this is basically
>              # "repo_contains_package"
>              package=$(find "${REPO_ISAR_DIR}"/"${DISTRO}" -name ${p##*/})
>              if [ -n "$package" ]; then
>                  cmp --silent "$package" "$p" && continue
>              fi
> -            sudo cp -n "${p}" "${pc}"
> +            sudo ln -P "${p}" "${pc}" 2>/dev/null || :
>          done
> +        sudo touch "${pc}/CACHEDIR.TAG"
>          sudo chown -R $(id -u):$(id -g) "${pc}"
>      '
>  }
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index 7822b14..166ec16 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -108,10 +108,10 @@ dpkg_runbuild() {
>          --chroot-setup-commands="echo \"Package: *\nPin: release
> n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \
>          --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" >
> /etc/apt/apt.conf.d/50isar-apt" \
>          --chroot-setup-commands="rm -f /var/log/dpkg.log" \
> -        --chroot-setup-commands="cp -n --no-preserve=owner
> ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \
> +        --chroot-setup-commands="ln -Pf ${ext_deb_dir}/*.deb -t ${deb_dir}/
> 2>/dev/null || :" \
>          --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-
> main-dummy_*.deb" \
> -        --finished-build-commands="cp -n --no-preserve=owner
> ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \
> -        --finished-build-commands="cp /var/log/dpkg.log
> ${ext_root}/dpkg_partial.log" \
> +        --finished-build-commands="ln -P ${deb_dir}/*.deb -t ${ext_deb_dir}/
> 2>/dev/null || :" \
> +        --finished-build-commands="cp /var/log/dpkg.log
> ${ext_root}/dpkg_partial.log; touch ${ext_deb_dir}/CACHEDIR.TAG" \
>          --debbuildopts="--source-option=-I" \
>          --build-dir=${WORKDIR} --dist="isar" ${DSC_FILE}
> 
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 786682d..147f517 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -78,7 +78,7 @@ BOOTSTRAP_SRC_${ROOTFS_ARCH} =
> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTF
> 
>  rootfs_prepare[weight] = "25"
>  rootfs_prepare(){
> -    sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> +    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>  }
> 
>  ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
> @@ -115,7 +115,6 @@ rootfs_configure_apt() {
>  EOSUDO
>  }
> 
> -
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>  rootfs_install_pkgs_update[weight] = "5"
>  rootfs_install_pkgs_update[isar-apt-lock] = "acquire-before"
> @@ -218,7 +217,7 @@ cache_deb_src() {
>      # Note: ISAR updates the apt state information(apt-get update) only once
> during bootstrap and
>      # relies on that through out the build. Copy that state information
> instead of apt-get update
>      # which generates a new state from upstream.
> -    sudo cp -Trpn "${BOOTSTRAP_SRC}/var/lib/apt/lists/"
> "${ROOTFSDIR}/var/lib/apt/lists/"
> +    sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/"
> "${ROOTFSDIR}/var/lib/apt/lists/"
> 
>      deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}
>      debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}
> @@ -308,7 +307,8 @@ rootfs_install_sstate_prepare() {
>      # so we use some mount magic to prevent that
>      mkdir -p ${WORKDIR}/mnt/rootfs
>      sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
> -    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar --one-file-system rootfs
> +    lopts="--one-file-system --exclude-caches --exclude=var/cache/apt/archives"

(several locations):
With the creation of CACHEDIR.TAG, and --exclude-caches, isn't the --exclude=
of that same path redundant?

Adriaan

> +    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar $lopts rootfs
>      sudo umount ${WORKDIR}/mnt/rootfs
>      sudo chown $(id -u):$(id -g) rootfs.tar
>  }
> diff --git a/meta/classes/sdk.bbclass b/meta/classes/sdk.bbclass
> index 79e01a1..3feada2 100644
> --- a/meta/classes/sdk.bbclass
> +++ b/meta/classes/sdk.bbclass
> @@ -82,7 +82,7 @@ do_rootfs_install[vardeps] += "${SDKROOTFSVARDEPS}"
>  ROOTFS_CONFIGURE_COMMAND_append_class-sdk = "
> ${@'rootfs_configure_isar_apt_dir' if d.getVar('SDK_INCLUDE_ISAR_APT') == '1'
> else ''}"
>  rootfs_configure_isar_apt_dir() {
>      # Copy isar-apt instead of mounting:
> -    sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
> +    sudo cp -Trpfx --reflink=auto ${REPO_ISAR_DIR}/${DISTRO}
> ${ROOTFSDIR}/isar-apt
>  }
> 
>  ROOTFS_POSTPROCESS_COMMAND_prepend_class-sdk = "sdkchroot_configscript "
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index b9ae16c..d53db4d 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -418,7 +418,8 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> 
>  bootstrap_sstate_prepare() {
>      # this runs in SSTATE_BUILDDIR, which will be deleted automatically
> -    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar --one-file-
> system $(basename "${ROOTFSDIR}")
> +    lopts="--one-file-system --exclude-caches --
> exclude=var/cache/apt/archives"
> +    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts
> $(basename "${ROOTFSDIR}")
>      sudo chown $(id -u):$(id -g) bootstrap.tar
>  }
> 
> --
> 2.34.1
> 
> --
> You received this message because you are subscribed to the Google Groups
> "isar-users" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to isar-users+unsubscribe@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/isar-users/20230104023128.374390-1-
> felix.moessbauer%40siemens.com.