Re: [PATCH] sstate: refactor to avoid intermediate files

[Henning Schild <henning.schild@siemens.com>]

Am Fri, 25 Feb 2022 12:51:30 +0300
schrieb Uladzimir Bely <ubely@ilbers.de>:

> In the email from Thursday, 24 February 2022 15:10:25 +03 user
> Adriaan Schmidt wrote:
> > By using the SSTATECREATEFUNCS and SSTATEPOSTINSTFUNCS
> > hooks (instead of sstate-interceptfuncs) we can avoid
> > the explicit creation of intermediate/temporary files
> > during sstate caching/restoring, which saves disk space
> > during build.
> > 
> > Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>
> > ---
> >  meta/classes/dpkg-base.bbclass                | 16 ++++++------
> >  meta/classes/rootfs.bbclass                   | 25
> > ++++++++++++------- .../isar-bootstrap/isar-bootstrap.inc         |
> > 22 +++++++++------- 3 files changed, 36 insertions(+), 27
> > deletions(-)
> > 
> > diff --git a/meta/classes/dpkg-base.bbclass
> > b/meta/classes/dpkg-base.bbclass index 928856a9..86933c57 100644
> > --- a/meta/classes/dpkg-base.bbclass
> > +++ b/meta/classes/dpkg-base.bbclass
> > @@ -237,27 +237,25 @@ python do_dpkg_build() {
> >  addtask dpkg_build
> > 
> >  SSTATETASKS += "do_dpkg_build"
> > -DPKG_SSTATE = "${WORKDIR}/dpkg-sstate"
> > -do_dpkg_build[dirs] += "${DPKG_SSTATE} ${S}/.."
> > -do_dpkg_build[cleandirs] += "${DPKG_SSTATE}"
> > -do_dpkg_build[sstate-plaindirs] = "${DPKG_SSTATE}"
> > -do_dpkg_build[sstate-interceptfuncs] = "dpkg_build_sstate_prepare"
> > +SSTATECREATEFUNCS += "dpkg_build_sstate_prepare"
> > +SSTATEPOSTINSTFUNCS += "dpkg_build_sstate_finalize"
> >   
> 
> I tested the patch and it looks and works well.
> 
> I also tried to find a way to reuse `sstate-install-<taskname>`
> directories which are deleted automatically and look much more
> "native" for sstate that our custom `rootfs-state` or
> `bootstrap-sstate` or `dpkg-sstate`, but didn't succeed, being not so
> much familiar with sstate.

I also tried that but really those are SSTATE_BUILDDIR and seem class
internal. I thought if i happen to choose a name that will later match
SSTATE_BUILDDIR the cleanup would be done only at the end.

But reading "sstate_package" one can see that the directly will be
cleaned also at the beginning.

https://github.com/ilbers/isar/blob/ffdd1b0ce026d21c8b62c06c926d205aad3078b6/meta/classes/sstate.bbclass#L650

So the better way seem to be the functions, which will execute in that
dir and the dir will be cleaned up.

> Reusing "standard" directories really reduces amount of code and
> makes things simple.

Those dirs seem clearly sstate internal with no way to hook in, or one
would need to change the sstate class upstream first.

Henning

> >  dpkg_build_sstate_prepare() {
> > +    # this runs in SSTATE_BUILDDIR, which will be deleted
> > automatically if [ -n "$(find ${S}/.. -maxdepth 1 -name '*.deb'
> > -print -quit)" ]; then -        ln -f ${S}/../*.deb -t
> > ${DPKG_SSTATE}
> > +        cp -f ${S}/../*.deb -t .
> >      fi
> >  }
> > 
> >  dpkg_build_sstate_finalize() {
> > -    if [ -n "$(find ${DPKG_SSTATE} -maxdepth 1 -name '*.deb'
> > -print -quit)" ]; then -        ln -f ${DPKG_SSTATE}/*.deb -t
> > ${S}/..
> > +    # this runs in SSTATE_INSTDIR
> > +    if [ -n "$(find . -maxdepth 1 -name '*.deb' -print -quit)" ];
> > then
> > +        mv -f ./*.deb -t ${S}/..
> >      fi
> >  }
> > 
> >  python do_dpkg_build_setscene() {
> >      sstate_setscene(d)
> > -    bb.build.exec_func('dpkg_build_sstate_finalize', d)
> >  }
> > 
> >  addtask dpkg_build_setscene
> > diff --git a/meta/classes/rootfs.bbclass
> > b/meta/classes/rootfs.bbclass index 2bdb3b6d..ba86c60a 100644
> > --- a/meta/classes/rootfs.bbclass
> > +++ b/meta/classes/rootfs.bbclass
> > @@ -289,26 +289,33 @@ addtask rootfs before do_build
> >  do_rootfs_postprocess[depends] = "base-apt:do_cache
> > isar-apt:do_cache_config"
> > 
> >  SSTATETASKS += "do_rootfs_install"
> > -ROOTFS_SSTATE = "${WORKDIR}/rootfs-sstate"
> > -do_rootfs_install[dirs] += "${ROOTFS_SSTATE} ${WORKDIR}/mnt/rootfs"
> > -do_rootfs_install[cleandirs] += "${ROOTFS_SSTATE}"
> > -do_rootfs_install[sstate-plaindirs] = "${ROOTFS_SSTATE}"
> > -do_rootfs_install[sstate-interceptfuncs] =
> > "rootfs_install_sstate_prepare" +SSTATECREATEFUNCS +=
> > "rootfs_install_sstate_prepare" +SSTATEPOSTINSTFUNCS +=
> > "rootfs_install_sstate_finalize"
> > 
> > -# the buildchroot is owned by root, so we need some sudoing to
> > pack and unpack +# the rootfs is owned by root, so we need some
> > sudoing to pack and unpack rootfs_install_sstate_prepare() {
> > +    # this runs in SSTATE_BUILDDIR, which will be deleted
> > automatically
> > +    # tar --one-file-system will cross bind-mounts to the same
> > filesystem,
> > +    # so we use some mount magic to prevent that
> > +    mkdir -p ${WORKDIR}/mnt/rootfs
> >      sudo mount --bind ${WORKDIR}/rootfs ${WORKDIR}/mnt/rootfs -o ro
> > -    sudo tar -C ${WORKDIR}/mnt -cpf ${ROOTFS_SSTATE}/rootfs.tar
> > --one-file-system rootfs +    sudo tar -C ${WORKDIR}/mnt -cpf
> > rootfs.tar --one-file-system rootfs sudo umount
> > ${WORKDIR}/mnt/rootfs
> > +    sudo chown $(id -u):$(id -g) rootfs.tar
> >  }
> >  do_rootfs_install_sstate_prepare[lockfiles] =
> > "${REPO_ISAR_DIR}/isar.lock"
> > 
> >  rootfs_install_sstate_finalize() {
> > -    sudo tar -C ${WORKDIR} -xpf ${ROOTFS_SSTATE}/rootfs.tar
> > +    # this runs in SSTATE_INSTDIR
> > +    # - after building the rootfs, the tar won't be there, but we
> > also don't need to unpack +    # - after restoring from cache,
> > there will be a tar which we unpack and then delete +    if [ -f
> > rootfs.tar ]; then
> > +        sudo tar -C ${WORKDIR} -xpf rootfs.tar
> > +        rm rootfs.tar
> > +    fi
> >  }
> > 
> >  python do_rootfs_install_setscene() {
> >      sstate_setscene(d)
> > -    bb.build.exec_func('rootfs_install_sstate_finalize', d)
> >  }
> >  addtask do_rootfs_install_setscene
> > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> > 2f483f5a..d6f90f63 100644
> > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > @@ -395,24 +395,28 @@ EOSUDO
> >  addtask bootstrap before do_build after do_generate_keyrings
> > 
> >  SSTATETASKS += "do_bootstrap"
> > -BOOTSTRAP_SSTATE = "${WORKDIR}/bootstrap-sstate"
> > -do_bootstrap[dirs] += "${BOOTSTRAP_SSTATE}"
> > -do_bootstrap[cleandirs] += "${BOOTSTRAP_SSTATE}"
> > -do_bootstrap[sstate-plaindirs] = "${BOOTSTRAP_SSTATE}"
> > -do_bootstrap[sstate-interceptfuncs] = "bootstrap_sstate_prepare"
> > +SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
> > +SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> > 
> >  bootstrap_sstate_prepare() {
> > -    sudo tar -C $(dirname "${ROOTFSDIR}") -cpf
> > ${BOOTSTRAP_SSTATE}/bootstrap.tar --one-file-system $(basename
> > "${ROOTFSDIR}") +    # this runs in SSTATE_BUILDDIR, which will be
> > deleted automatically +    sudo tar -C $(dirname "${ROOTFSDIR}")
> > -cpf bootstrap.tar --one-file-system $(basename "${ROOTFSDIR}") +
> >  sudo chown $(id -u):$(id -g) bootstrap.tar
> >  }
> > 
> >  bootstrap_sstate_finalize() {
> > -    sudo tar -C $(dirname "${ROOTFSDIR}") -xpf
> > ${BOOTSTRAP_SSTATE}/bootstrap.tar -    sudo ln -Tfsr "${ROOTFSDIR}"
> > "${DEPLOY_ISAR_BOOTSTRAP}"
> > +    # this runs in SSTATE_INSTDIR
> > +    # - after building the bootstrap, the tar won't be there, but
> > we also don't need to unpack +    # - after restoring from cache,
> > there will be a tar which we unpack and then delete +    if [ -f
> > bootstrap.tar ]; then
> > +        sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
> > +        sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> > +        rm bootstrap.tar
> > +    fi
> >  }
> > 
> >  python do_bootstrap_setscene() {
> >      sstate_setscene(d)
> > -    bb.build.exec_func('bootstrap_sstate_finalize', d)
> >  }
> > 
> >  addtask do_bootstrap_setscene  
> 
>