Re: [PATCH] sstate: refactor to avoid intermediate files

[Uladzimir Bely <ubely@ilbers.de>]

In the email from Thursday, 24 February 2022 15:10:25 +03 user Adriaan Schmidt 
wrote:
> By using the SSTATECREATEFUNCS and SSTATEPOSTINSTFUNCS
> hooks (instead of sstate-interceptfuncs) we can avoid
> the explicit creation of intermediate/temporary files
> during sstate caching/restoring, which saves disk space
> during build.
> 
> Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>
> ---
>  meta/classes/dpkg-base.bbclass                | 16 ++++++------
>  meta/classes/rootfs.bbclass                   | 25 ++++++++++++-------
>  .../isar-bootstrap/isar-bootstrap.inc         | 22 +++++++++-------
>  3 files changed, 36 insertions(+), 27 deletions(-)
> 
> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
> index 928856a9..86933c57 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -237,27 +237,25 @@ python do_dpkg_build() {
>  addtask dpkg_build
> 
>  SSTATETASKS += "do_dpkg_build"
> -DPKG_SSTATE = "${WORKDIR}/dpkg-sstate"
> -do_dpkg_build[dirs] += "${DPKG_SSTATE} ${S}/.."
> -do_dpkg_build[cleandirs] += "${DPKG_SSTATE}"
> -do_dpkg_build[sstate-plaindirs] = "${DPKG_SSTATE}"
> -do_dpkg_build[sstate-interceptfuncs] = "dpkg_build_sstate_prepare"
> +SSTATECREATEFUNCS += "dpkg_build_sstate_prepare"
> +SSTATEPOSTINSTFUNCS += "dpkg_build_sstate_finalize"
> 

I tested the patch and it looks and works well.

I also tried to find a way to reuse `sstate-install-<taskname>` directories 
which are deleted automatically and look much more "native" for sstate that 
our custom `rootfs-state` or `bootstrap-sstate` or `dpkg-sstate`, but didn't 
succeed, being not so much familiar with sstate.

Reusing "standard" directories really reduces amount of code and makes things 
simple.

>  dpkg_build_sstate_prepare() {
> +    # this runs in SSTATE_BUILDDIR, which will be deleted automatically
>      if [ -n "$(find ${S}/.. -maxdepth 1 -name '*.deb' -print -quit)" ];
> then -        ln -f ${S}/../*.deb -t ${DPKG_SSTATE}
> +        cp -f ${S}/../*.deb -t .
>      fi
>  }
> 
>  dpkg_build_sstate_finalize() {
> -    if [ -n "$(find ${DPKG_SSTATE} -maxdepth 1 -name '*.deb' -print -quit)"
> ]; then -        ln -f ${DPKG_SSTATE}/*.deb -t ${S}/..
> +    # this runs in SSTATE_INSTDIR
> +    if [ -n "$(find . -maxdepth 1 -name '*.deb' -print -quit)" ]; then
> +        mv -f ./*.deb -t ${S}/..
>      fi
>  }
> 
>  python do_dpkg_build_setscene() {
>      sstate_setscene(d)
> -    bb.build.exec_func('dpkg_build_sstate_finalize', d)
>  }
> 
>  addtask dpkg_build_setscene
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 2bdb3b6d..ba86c60a 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -289,26 +289,33 @@ addtask rootfs before do_build
>  do_rootfs_postprocess[depends] = "base-apt:do_cache
> isar-apt:do_cache_config"
> 
>  SSTATETASKS += "do_rootfs_install"
> -ROOTFS_SSTATE = "${WORKDIR}/rootfs-sstate"
> -do_rootfs_install[dirs] += "${ROOTFS_SSTATE} ${WORKDIR}/mnt/rootfs"
> -do_rootfs_install[cleandirs] += "${ROOTFS_SSTATE}"
> -do_rootfs_install[sstate-plaindirs] = "${ROOTFS_SSTATE}"
> -do_rootfs_install[sstate-interceptfuncs] = "rootfs_install_sstate_prepare"
> +SSTATECREATEFUNCS += "rootfs_install_sstate_prepare"
> +SSTATEPOSTINSTFUNCS += "rootfs_install_sstate_finalize"
> 
> -# the buildchroot is owned by root, so we need some sudoing to pack and
> unpack +# the rootfs is owned by root, so we need some sudoing to pack and
> unpack rootfs_install_sstate_prepare() {
> +    # this runs in SSTATE_BUILDDIR, which will be deleted automatically
> +    # tar --one-file-system will cross bind-mounts to the same filesystem,
> +    # so we use some mount magic to prevent that
> +    mkdir -p ${WORKDIR}/mnt/rootfs
>      sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
> -    sudo tar -C ${WORKDIR}/mnt -cpf ${ROOTFS_SSTATE}/rootfs.tar
> --one-file-system rootfs +    sudo tar -C ${WORKDIR}/mnt -cpf rootfs.tar
> --one-file-system rootfs sudo umount ${WORKDIR}/mnt/rootfs
> +    sudo chown $(id -u):$(id -g) rootfs.tar
>  }
>  do_rootfs_install_sstate_prepare[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
> 
>  rootfs_install_sstate_finalize() {
> -    sudo tar -C ${WORKDIR} -xpf ${ROOTFS_SSTATE}/rootfs.tar
> +    # this runs in SSTATE_INSTDIR
> +    # - after building the rootfs, the tar won't be there, but we also
> don't need to unpack +    # - after restoring from cache, there will be a
> tar which we unpack and then delete +    if [ -f rootfs.tar ]; then
> +        sudo tar -C ${WORKDIR} -xpf rootfs.tar
> +        rm rootfs.tar
> +    fi
>  }
> 
>  python do_rootfs_install_setscene() {
>      sstate_setscene(d)
> -    bb.build.exec_func('rootfs_install_sstate_finalize', d)
>  }
>  addtask do_rootfs_install_setscene
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> 2f483f5a..d6f90f63 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -395,24 +395,28 @@ EOSUDO
>  addtask bootstrap before do_build after do_generate_keyrings
> 
>  SSTATETASKS += "do_bootstrap"
> -BOOTSTRAP_SSTATE = "${WORKDIR}/bootstrap-sstate"
> -do_bootstrap[dirs] += "${BOOTSTRAP_SSTATE}"
> -do_bootstrap[cleandirs] += "${BOOTSTRAP_SSTATE}"
> -do_bootstrap[sstate-plaindirs] = "${BOOTSTRAP_SSTATE}"
> -do_bootstrap[sstate-interceptfuncs] = "bootstrap_sstate_prepare"
> +SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
> +SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> 
>  bootstrap_sstate_prepare() {
> -    sudo tar -C $(dirname "${ROOTFSDIR}") -cpf
> ${BOOTSTRAP_SSTATE}/bootstrap.tar --one-file-system $(basename
> "${ROOTFSDIR}") +    # this runs in SSTATE_BUILDDIR, which will be deleted
> automatically +    sudo tar -C $(dirname "${ROOTFSDIR}") -cpf bootstrap.tar
> --one-file-system $(basename "${ROOTFSDIR}") +    sudo chown $(id -u):$(id
> -g) bootstrap.tar
>  }
> 
>  bootstrap_sstate_finalize() {
> -    sudo tar -C $(dirname "${ROOTFSDIR}") -xpf
> ${BOOTSTRAP_SSTATE}/bootstrap.tar -    sudo ln -Tfsr "${ROOTFSDIR}"
> "${DEPLOY_ISAR_BOOTSTRAP}"
> +    # this runs in SSTATE_INSTDIR
> +    # - after building the bootstrap, the tar won't be there, but we also
> don't need to unpack +    # - after restoring from cache, there will be a
> tar which we unpack and then delete +    if [ -f bootstrap.tar ]; then
> +        sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
> +        sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> +        rm bootstrap.tar
> +    fi
>  }
> 
>  python do_bootstrap_setscene() {
>      sstate_setscene(d)
> -    bb.build.exec_func('bootstrap_sstate_finalize', d)
>  }
> 
>  addtask do_bootstrap_setscene


-- 
Uladzimir Bely