[PATCH 1/1] Reduce disk usage during build by avoiding copies of apt cache

[Felix Moessbauer <felix.moessbauer@siemens.com>]

From: "Roberto A. Foglietta" <roberto.foglietta@gmail.com>

This patch leverages few ways to accomplish the task to optimise the large
buildings. It uses hard physical links instead of copying files which is a
faster way and saves disk spaces. Put a CACHEDIR.TAG file in each cache folder
in such a way every tar that uses the option --exclude-caches could be avoided
to include debian packages saving a lot of time and disk space.

Signed-off-by: Roberto A. Foglietta <roberto.foglietta@gmail.com>
Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---

This is a cleaned up and squashed version of the following commits
from Robertos ISAR fork:

- 83ce56580c39a2f708f9a3ede73439c82edac1be
- 02bfafb20a8c43585561b9de306a2e23c5ff50bb

In addition, I applied some small fixes to fix issues on rebuild.
This is a compromise between the large patch series of Roberto and
the too minimalistic one of Uladzimir. As the series from Roberto is
not on this list, I want to send out this patch to have something which
we can review.

Maybe we can integrate that first to solve 90% of the issue and then
continue improving it.

Best regards,
Felix

 meta/classes/deb-dl-dir.bbclass                     | 11 ++++++-----
 meta/classes/dpkg.bbclass                           |  6 +++---
 meta/classes/rootfs.bbclass                         |  8 ++++----
 meta/classes/sdk.bbclass                            |  2 +-
 meta/recipes-core/isar-bootstrap/isar-bootstrap.inc |  3 ++-
 5 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 3b1517d..3a87390 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -84,9 +84,9 @@ deb_dl_dir_import() {
     flock -s "${pc}".lock -c '
         set -e
         printenv | grep -q BB_VERBOSE_LOGS && set -x
-
+        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
         sudo find "${pc}" -type f -iname "*\.deb" -exec \
-            cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ {} +
+            ln -Pf -t "${rootfs}"/var/cache/apt/archives/ {} + 2>/dev/null || :
     '
 }
 
@@ -97,20 +97,21 @@ deb_dl_dir_export() {
     flock "${pc}".lock -c '
         set -e
         printenv | grep -q BB_VERBOSE_LOGS && set -x
-
+        sudo touch "${rootfs}"/var/cache/apt/archives/CACHEDIR.TAG
         find "${rootfs}"/var/cache/apt/archives/ \
             -maxdepth 1 -type f -iname '*\.deb' |\
         while read p; do
             # skip files from a previous export
-            [ -f "${pc}/${p##*/}" ] && continue
+            [ -e "${pc}/${p##*/}" ] && continue
             # can not reuse bitbake function here, this is basically
             # "repo_contains_package"
             package=$(find "${REPO_ISAR_DIR}"/"${DISTRO}" -name ${p##*/})
             if [ -n "$package" ]; then
                 cmp --silent "$package" "$p" && continue
             fi
-            sudo cp -n "${p}" "${pc}"
+            sudo ln -P "${p}" "${pc}" 2>/dev/null || :
         done
+        sudo touch "${pc}/CACHEDIR.TAG"
         sudo chown -R $(id -u):$(id -g) "${pc}"
     '
 }
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7822b14..166ec16 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -108,10 +108,10 @@ dpkg_runbuild() {
         --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \
         --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
-        --chroot-setup-commands="cp -n --no-preserve=owner ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \
+        --chroot-setup-commands="ln -Pf ${ext_deb_dir}/*.deb -t ${deb_dir}/ 2>/dev/null || :" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
-        --finished-build-commands="cp -n --no-preserve=owner ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \
-        --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
+        --finished-build-commands="ln -P ${deb_dir}/*.deb -t ${ext_deb_dir}/ 2>/dev/null || :" \
+        --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log; touch ${ext_deb_dir}/CACHEDIR.TAG" \
         --debbuildopts="--source-option=-I" \
         --build-dir=${WORKDIR} --dist="isar" ${DSC_FILE}
 
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 786682d..147f517 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -78,7 +78,7 @@ BOOTSTRAP_SRC_${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTF
 
 rootfs_prepare[weight] = "25"
 rootfs_prepare(){
-    sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+    sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
 }
 
 ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -115,7 +115,6 @@ rootfs_configure_apt() {
 EOSUDO
 }
 
-
 ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
 rootfs_install_pkgs_update[weight] = "5"
 rootfs_install_pkgs_update[isar-apt-lock] = "acquire-before"
@@ -218,7 +217,7 @@ cache_deb_src() {
     # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
     # relies on that through out the build. Copy that state information instead of apt-get update
     # which generates a new state from upstream.
-    sudo cp -Trpn "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+    sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
 
     deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
     debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
@@ -308,7 +307,8 @@ rootfs_install_sstate_prepare() {
     # so we use some mount magic to prevent that
     mkdir -p ${WORKDIR}/mnt/rootfs
     sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
-    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar --one-file-system rootfs
+    lopts="--one-file-system --exclude-caches --exclude=var/cache/apt/archives"
+    sudo tar -C ${WORKDIR}/mnt -cpSf rootfs.tar $lopts rootfs
     sudo umount ${WORKDIR}/mnt/rootfs
     sudo chown $(id -u):$(id -g) rootfs.tar
 }
diff --git a/meta/classes/sdk.bbclass b/meta/classes/sdk.bbclass
index 79e01a1..3feada2 100644
--- a/meta/classes/sdk.bbclass
+++ b/meta/classes/sdk.bbclass
@@ -82,7 +82,7 @@ do_rootfs_install[vardeps] += "${SDKROOTFSVARDEPS}"
 ROOTFS_CONFIGURE_COMMAND_append_class-sdk = " ${@'rootfs_configure_isar_apt_dir' if d.getVar('SDK_INCLUDE_ISAR_APT') == '1' else ''}"
 rootfs_configure_isar_apt_dir() {
     # Copy isar-apt instead of mounting:
-    sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
+    sudo cp -Trpfx --reflink=auto ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
 }
 
 ROOTFS_POSTPROCESS_COMMAND_prepend_class-sdk = "sdkchroot_configscript "
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index b9ae16c..d53db4d 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -418,7 +418,8 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
 
 bootstrap_sstate_prepare() {
     # this runs in SSTATE_BUILDDIR, which will be deleted automatically
-    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar --one-file-system $(basename "${ROOTFSDIR}")
+    lopts="--one-file-system --exclude-caches --exclude=var/cache/apt/archives"
+    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
     sudo chown $(id -u):$(id -g) bootstrap.tar
 }
 
-- 
2.34.1