public inbox for isar-users@googlegroups.com
 help / color / mirror / Atom feed
* [PATCH] sstate: refactor to avoid intermediate files
@ 2022-02-24 12:10 Adriaan Schmidt
  2022-02-25  9:51 ` Uladzimir Bely
  2022-03-03 10:23 ` Anton Mikanovich
  0 siblings, 2 replies; 4+ messages in thread
From: Adriaan Schmidt @ 2022-02-24 12:10 UTC (permalink / raw)
  To: isar-users; +Cc: Adriaan Schmidt

By using the SSTATECREATEFUNCS and SSTATEPOSTINSTFUNCS
hooks (instead of sstate-interceptfuncs) we can avoid
the explicit creation of intermediate/temporary files
during sstate caching/restoring, which saves disk space
during build.

Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>
---
 meta/classes/dpkg-base.bbclass                | 16 ++++++------
 meta/classes/rootfs.bbclass                   | 25 ++++++++++++-------
 .../isar-bootstrap/isar-bootstrap.inc         | 22 +++++++++-------
 3 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 928856a9..86933c57 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -237,27 +237,25 @@ python do_dpkg_build() {
 addtask dpkg_build
 
 SSTATETASKS += "do_dpkg_build"
-DPKG_SSTATE = "${WORKDIR}/dpkg-sstate"
-do_dpkg_build[dirs] += "${DPKG_SSTATE} ${S}/.."
-do_dpkg_build[cleandirs] += "${DPKG_SSTATE}"
-do_dpkg_build[sstate-plaindirs] = "${DPKG_SSTATE}"
-do_dpkg_build[sstate-interceptfuncs] = "dpkg_build_sstate_prepare"
+SSTATECREATEFUNCS += "dpkg_build_sstate_prepare"
+SSTATEPOSTINSTFUNCS += "dpkg_build_sstate_finalize"
 
 dpkg_build_sstate_prepare() {
+    # this runs in SSTATE_BUILDDIR, which will be deleted automatically
     if [ -n "$(find ${S}/.. -maxdepth 1 -name '*.deb' -print -quit)" ]; then
-        ln -f ${S}/../*.deb -t ${DPKG_SSTATE}
+        cp -f ${S}/../*.deb -t .
     fi
 }
 
 dpkg_build_sstate_finalize() {
-    if [ -n "$(find ${DPKG_SSTATE} -maxdepth 1 -name '*.deb' -print -quit)" ]; then
-        ln -f ${DPKG_SSTATE}/*.deb -t ${S}/..
+    # this runs in SSTATE_INSTDIR
+    if [ -n "$(find . -maxdepth 1 -name '*.deb' -print -quit)" ]; then
+        mv -f ./*.deb -t ${S}/..
     fi
 }
 
 python do_dpkg_build_setscene() {
     sstate_setscene(d)
-    bb.build.exec_func('dpkg_build_sstate_finalize', d)
 }
 
 addtask dpkg_build_setscene
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 2bdb3b6d..ba86c60a 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -289,26 +289,33 @@ addtask rootfs before do_build
 do_rootfs_postprocess[depends] = "base-apt:do_cache isar-apt:do_cache_config"
 
 SSTATETASKS += "do_rootfs_install"
-ROOTFS_SSTATE = "${WORKDIR}/rootfs-sstate"
-do_rootfs_install[dirs] += "${ROOTFS_SSTATE} ${WORKDIR}/mnt/rootfs"
-do_rootfs_install[cleandirs] += "${ROOTFS_SSTATE}"
-do_rootfs_install[sstate-plaindirs] = "${ROOTFS_SSTATE}"
-do_rootfs_install[sstate-interceptfuncs] = "rootfs_install_sstate_prepare"
+SSTATECREATEFUNCS += "rootfs_install_sstate_prepare"
+SSTATEPOSTINSTFUNCS += "rootfs_install_sstate_finalize"
 
-# the buildchroot is owned by root, so we need some sudoing to pack and unpack
+# the rootfs is owned by root, so we need some sudoing to pack and unpack
 rootfs_install_sstate_prepare() {
+    # this runs in SSTATE_BUILDDIR, which will be deleted automatically
+    # tar --one-file-system will cross bind-mounts to the same filesystem,
+    # so we use some mount magic to prevent that
+    mkdir -p ${WORKDIR}/mnt/rootfs
     sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
-    sudo tar -C ${WORKDIR}/mnt -cpf ${ROOTFS_SSTATE}/rootfs.tar --one-file-system rootfs
+    sudo tar -C ${WORKDIR}/mnt -cpf rootfs.tar --one-file-system rootfs
     sudo umount ${WORKDIR}/mnt/rootfs
+    sudo chown $(id -u):$(id -g) rootfs.tar
 }
 do_rootfs_install_sstate_prepare[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
 
 rootfs_install_sstate_finalize() {
-    sudo tar -C ${WORKDIR} -xpf ${ROOTFS_SSTATE}/rootfs.tar
+    # this runs in SSTATE_INSTDIR
+    # - after building the rootfs, the tar won't be there, but we also don't need to unpack
+    # - after restoring from cache, there will be a tar which we unpack and then delete
+    if [ -f rootfs.tar ]; then
+        sudo tar -C ${WORKDIR} -xpf rootfs.tar
+        rm rootfs.tar
+    fi
 }
 
 python do_rootfs_install_setscene() {
     sstate_setscene(d)
-    bb.build.exec_func('rootfs_install_sstate_finalize', d)
 }
 addtask do_rootfs_install_setscene
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2f483f5a..d6f90f63 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -395,24 +395,28 @@ EOSUDO
 addtask bootstrap before do_build after do_generate_keyrings
 
 SSTATETASKS += "do_bootstrap"
-BOOTSTRAP_SSTATE = "${WORKDIR}/bootstrap-sstate"
-do_bootstrap[dirs] += "${BOOTSTRAP_SSTATE}"
-do_bootstrap[cleandirs] += "${BOOTSTRAP_SSTATE}"
-do_bootstrap[sstate-plaindirs] = "${BOOTSTRAP_SSTATE}"
-do_bootstrap[sstate-interceptfuncs] = "bootstrap_sstate_prepare"
+SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
+SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
 
 bootstrap_sstate_prepare() {
-    sudo tar -C $(dirname "${ROOTFSDIR}") -cpf ${BOOTSTRAP_SSTATE}/bootstrap.tar --one-file-system $(basename "${ROOTFSDIR}")
+    # this runs in SSTATE_BUILDDIR, which will be deleted automatically
+    sudo tar -C $(dirname "${ROOTFSDIR}") -cpf bootstrap.tar --one-file-system $(basename "${ROOTFSDIR}")
+    sudo chown $(id -u):$(id -g) bootstrap.tar
 }
 
 bootstrap_sstate_finalize() {
-    sudo tar -C $(dirname "${ROOTFSDIR}") -xpf ${BOOTSTRAP_SSTATE}/bootstrap.tar
-    sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+    # this runs in SSTATE_INSTDIR
+    # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
+    # - after restoring from cache, there will be a tar which we unpack and then delete
+    if [ -f bootstrap.tar ]; then
+        sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
+        sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+        rm bootstrap.tar
+    fi
 }
 
 python do_bootstrap_setscene() {
     sstate_setscene(d)
-    bb.build.exec_func('bootstrap_sstate_finalize', d)
 }
 
 addtask do_bootstrap_setscene
-- 
2.30.2


^ permalink raw reply	[flat|nested] 4+ messages in thread
end of thread, other threads:[~2022-03-03 10:23 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-02-24 12:10 [PATCH] sstate: refactor to avoid intermediate files Adriaan Schmidt
2022-02-25  9:51 ` Uladzimir Bely
2022-02-25 10:02   ` Henning Schild
2022-03-03 10:23 ` Anton Mikanovich

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox