[PATCH 1/2] add support for derived sbuild chroots

[PATCH 1/2] add support for derived sbuild chroots

[Felix Moessbauer]

This patch adds support to create derived sbuild chroots to speedup the
build process. For packages that share a large set of common build
dependencies, a derived sbuild chroot can be created to avoid the
overhead of installing all base build-deps on each sbuild invocation.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 doc/user_manual.md                            | 27 +++++++++++++++++++
 meta/classes/crossvars.bbclass                | 11 +++++---
 .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/doc/user_manual.md b/doc/user_manual.md
index 92075a88..e7a16e78 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -1348,3 +1348,30 @@ To enable it, add the below line to your local.conf file.
 ```
 BASE_REPO_FEATURES = "cache-deb-src"
 ```
+
+## Use a custom sbuild chroot to speedup build
+
+### Motivation
+
+There are use-cases, where many packages need to be compiled but all of them
+need a similar base of build dependencies. In case the baseline is quite big,
+this adds a significant overhead as the build dependencies are installed individually
+for each and every package.
+
+### Solution
+
+By creating a dedicated sbuild chroot for this use-case, the baseline can be installed
+first and then all package builds of this type can use it. For that, create a
+new recipe with the name `sbuild-chroot-<host|target>-<flavor>`. In that recipe,
+define the following:
+
+```
+require recipes-devtools/sbuild-chroot/sbuild-chroot-<host|target>.bb
+
+SBUILD_FLAVOR = "<your flavor, e.g. clang>"
+SBUILD_CHROOT_PREINSTALL_COMMON += "<base packages>"
+```
+
+Then, in the dpkg recipe of your package, simply set `SBUILD_FLAVOR = "<your flavor>"`.
+To install additional packages into the sbuild chroot, add them to `SBUILD_CHROOT_PREINSTALL_COMMON`.
+An full example is provided by the `samefile` recipe.
diff --git a/meta/classes/crossvars.bbclass b/meta/classes/crossvars.bbclass
index 201d460f..8d1da09c 100644
--- a/meta/classes/crossvars.bbclass
+++ b/meta/classes/crossvars.bbclass
@@ -10,22 +10,27 @@ python __anonymous() {
     d.setVar('SCHROOT_USER', pwd.getpwuid(os.geteuid()).pw_name)
 
     mode = d.getVar('ISAR_CROSS_COMPILE')
+
+    # support derived schroots
+    flavor = d.getVar('SBUILD_FLAVOR')
+    flavor_suffix = ('-' + flavor) if flavor else ''
+
     distro_arch = d.getVar('DISTRO_ARCH')
     if mode == "0" or d.getVar('HOST_ARCH') == distro_arch or distro_arch == None:
         d.setVar('BUILD_HOST_ARCH', distro_arch)
         schroot_dir = d.getVar('SCHROOT_TARGET_DIR', False)
-        sbuild_dep = "sbuild-chroot-target:do_build"
+        sbuild_dep = "sbuild-chroot-target" + flavor_suffix + ":do_build"
         buildchroot_dir = d.getVar('BUILDCHROOT_TARGET_DIR', False)
         buildchroot_dep = "buildchroot-target:do_build"
         sdk_toolchain = "build-essential"
     else:
         d.setVar('BUILD_HOST_ARCH', d.getVar('HOST_ARCH'))
         schroot_dir = d.getVar('SCHROOT_HOST_DIR', False)
-        sbuild_dep = "sbuild-chroot-host:do_build"
+        sbuild_dep = "sbuild-chroot-host" + flavor_suffix + ":do_build"
         buildchroot_dir = d.getVar('BUILDCHROOT_HOST_DIR', False)
         buildchroot_dep = "buildchroot-host:do_build"
         sdk_toolchain = "crossbuild-essential-" + distro_arch
-    d.setVar('SCHROOT_DIR', schroot_dir)
+    d.setVar('SCHROOT_DIR', schroot_dir + flavor_suffix)
     d.setVar('SCHROOT_DEP', sbuild_dep)
     d.setVar('BUILDCHROOT_DIR', buildchroot_dir)
     d.setVar('BUILDCHROOT_DEP', buildchroot_dep)
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
index fd8bb648..b786eef5 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
@@ -11,6 +11,11 @@ PV = "1.0"
 inherit rootfs
 inherit compat
 
+# set the flavor to create derived sbuild chroots
+# this maps to a schroot created by a recipe named sbuild-chroot-<host|target>-<flavor>
+SBUILD_FLAVOR ??= ""
+SBUILD_SCHROOT_SUFFIX = "${@ ('-' + d.getVar('SBUILD_FLAVOR')) if d.getVar('SBUILD_FLAVOR') else ''}"
+
 python() {
     distro_gcc = d.getVar('DISTRO_GCC')
     distro_arch = d.getVar('DISTRO_ARCH')
@@ -52,7 +57,7 @@ ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}"
 # We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild
 ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt"
 
-DEPLOY_SCHROOT = "${@d.getVar('SCHROOT_' + d.getVar('SBUILD_VARIANT').upper() + '_DIR')}"
+DEPLOY_SCHROOT = "${@d.getVar('SCHROOT_' + d.getVar('SBUILD_VARIANT').upper() + '_DIR')}${SBUILD_SCHROOT_SUFFIX}"
 
 do_sbuildchroot_deploy[dirs] = "${DEPLOY_DIR}/schroot-${SBUILD_VARIANT}"
 do_sbuildchroot_deploy() {
-- 
2.34.1

[PATCH 2/2] add example how to use derived sbuild chroots

[Felix Moessbauer]

This patch extends the samefile recipe to use a derived sbuild chroot to
compile the tool using clang instead of gcc. While there are easier ways
to achive this goal, it is still valuable to demonstrate how to use the
infrastructure.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta-isar/recipes-app/samefile/samefile_2.14.bb       |  8 ++++++++
 .../sbuild-chroot/sbuild-chroot-target-clang.bb       | 11 +++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb

diff --git a/meta-isar/recipes-app/samefile/samefile_2.14.bb b/meta-isar/recipes-app/samefile/samefile_2.14.bb
index c53c9445..989e1983 100644
--- a/meta-isar/recipes-app/samefile/samefile_2.14.bb
+++ b/meta-isar/recipes-app/samefile/samefile_2.14.bb
@@ -5,6 +5,9 @@
 
 inherit dpkg
 
+# for demo reasons, we compile with clang if not cross-compiling
+SBUILD_FLAVOR = "${@ 'clang' if d.getVar('ISAR_CROSS_COMPILE') != '1' else '' }"
+
 DEBIAN_DEPENDS = "\${misc:Depends}"
 DESCRIPTION = "utility that finds files with identical contents"
 
@@ -42,4 +45,9 @@ EOF
 
     # We can also customize afterwards, in this case change the package section.
     sed -i -e 's/Section: misc/Section: utils/g' ${S}/debian/control
+
+    # use clang to compile (no cross support yet)
+    if [ "${SBUILD_FLAVOR}" = "clang" ]; then
+        sed -i -e 's/\(#\!.*\)/\1\nexport CC=clang/g' ${S}/debian/rules
+    fi
 }
diff --git a/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
new file mode 100644
index 00000000..3df16368
--- /dev/null
+++ b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
@@ -0,0 +1,11 @@
+# Root filesystem for packages building with clang
+#
+# This software is a part of ISAR.
+# Copyright (C) 2023 Siemens AG
+
+DESCRIPTION = "Isar sbuild/schroot filesystem for target (clang variant)"
+
+require recipes-devtools/sbuild-chroot/sbuild-chroot-target.bb
+
+SBUILD_FLAVOR = "clang"
+SBUILD_CHROOT_PREINSTALL_COMMON += "clang"
-- 
2.34.1

Re: [PATCH 2/2] add example how to use derived sbuild chroots

[Jan Kiszka]

On 13.04.23 09:00, Felix Moessbauer wrote:
> This patch extends the samefile recipe to use a derived sbuild chroot to
> compile the tool using clang instead of gcc. While there are easier ways
> to achive this goal, it is still valuable to demonstrate how to use the
> infrastructure.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  meta-isar/recipes-app/samefile/samefile_2.14.bb       |  8 ++++++++
>  .../sbuild-chroot/sbuild-chroot-target-clang.bb       | 11 +++++++++++
>  2 files changed, 19 insertions(+)
>  create mode 100644 meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> 
> diff --git a/meta-isar/recipes-app/samefile/samefile_2.14.bb b/meta-isar/recipes-app/samefile/samefile_2.14.bb
> index c53c9445..989e1983 100644
> --- a/meta-isar/recipes-app/samefile/samefile_2.14.bb
> +++ b/meta-isar/recipes-app/samefile/samefile_2.14.bb
> @@ -5,6 +5,9 @@
>  
>  inherit dpkg
>  
> +# for demo reasons, we compile with clang if not cross-compiling
> +SBUILD_FLAVOR = "${@ 'clang' if d.getVar('ISAR_CROSS_COMPILE') != '1' else '' }"
> +
>  DEBIAN_DEPENDS = "\${misc:Depends}"
>  DESCRIPTION = "utility that finds files with identical contents"
>  
> @@ -42,4 +45,9 @@ EOF
>  
>      # We can also customize afterwards, in this case change the package section.
>      sed -i -e 's/Section: misc/Section: utils/g' ${S}/debian/control
> +
> +    # use clang to compile (no cross support yet)
> +    if [ "${SBUILD_FLAVOR}" = "clang" ]; then
> +        sed -i -e 's/\(#\!.*\)/\1\nexport CC=clang/g' ${S}/debian/rules
> +    fi
>  }
> diff --git a/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> new file mode 100644
> index 00000000..3df16368
> --- /dev/null
> +++ b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> @@ -0,0 +1,11 @@
> +# Root filesystem for packages building with clang
> +#
> +# This software is a part of ISAR.
> +# Copyright (C) 2023 Siemens AG
> +
> +DESCRIPTION = "Isar sbuild/schroot filesystem for target (clang variant)"
> +
> +require recipes-devtools/sbuild-chroot/sbuild-chroot-target.bb
> +
> +SBUILD_FLAVOR = "clang"

Could this be auto-derived from a helper-include by taking the recipe
name and sub-tracting a conventional prefix?

Jan

> +SBUILD_CHROOT_PREINSTALL_COMMON += "clang"

-- 
Siemens AG, Technology
Competence Center Embedded Linux

Re: [PATCH 1/2] add support for derived sbuild chroots

[Uladzimir Bely]

On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-users
wrote:
> This patch adds support to create derived sbuild chroots to speedup
> the
> build process. For packages that share a large set of common build
> dependencies, a derived sbuild chroot can be created to avoid the
> overhead of installing all base build-deps on each sbuild invocation.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  doc/user_manual.md                            | 27
> +++++++++++++++++++
>  meta/classes/crossvars.bbclass                | 11 +++++---
>  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
>  3 files changed, 41 insertions(+), 4 deletions(-)

Despite passing CI, patch 2 has still open question. Should we merge it
or there some v2 expected?

Re: [PATCH 1/2] add support for derived sbuild chroots

[Uladzimir Bely]

On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-users
wrote:
> This patch adds support to create derived sbuild chroots to speedup
> the
> build process. For packages that share a large set of common build
> dependencies, a derived sbuild chroot can be created to avoid the
> overhead of installing all base build-deps on each sbuild invocation.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  doc/user_manual.md                            | 27
> +++++++++++++++++++
>  meta/classes/crossvars.bbclass                | 11 +++++---
>  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
>  3 files changed, 41 insertions(+), 4 deletions(-)

We are going to merge this in near few days in spite of an open
question left in discussion. The reasons:

1. It passes internal CI (dev/fast/full).

2. It was useful in case of 'meta-iot2050' downstream for rewriting
"npm.bbclass" in order not to use buildchroot.bbclass (that is going to
be deleted with "Imager schroot migration" patchset that is to be
merged soon too.

Re: [PATCH 1/2] add support for derived sbuild chroots

[Jan Kiszka]

On 13.06.23 08:24, Uladzimir Bely wrote:
> On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-users
> wrote:
>> This patch adds support to create derived sbuild chroots to speedup
>> the
>> build process. For packages that share a large set of common build
>> dependencies, a derived sbuild chroot can be created to avoid the
>> overhead of installing all base build-deps on each sbuild invocation.
>>
>> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
>> ---
>>  doc/user_manual.md                            | 27
>> +++++++++++++++++++
>>  meta/classes/crossvars.bbclass                | 11 +++++---
>>  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
>>  3 files changed, 41 insertions(+), 4 deletions(-)
> 
> We are going to merge this in near few days in spite of an open
> question left in discussion. The reasons:
> 
> 1. It passes internal CI (dev/fast/full).
> 
> 2. It was useful in case of 'meta-iot2050' downstream for rewriting
> "npm.bbclass" in order not to use buildchroot.bbclass (that is going to
> be deleted with "Imager schroot migration" patchset that is to be
> merged soon too.

To track what I discussed with Baurzhan offlist: I would be good to have
some QA check running when using a derived sbuild chroot that the
package built has all extra packages as part of its Build-Depends so
that we are not create again silently broken debianizations. Any ideas
how to achieve that best, considering also implicit inclusions of the
Build-Depends?

Furthermore, I had a small comment on patch 2.

Jan

-- 
Siemens AG, Technology
Competence Center Embedded Linux

Re: [PATCH 1/2] add support for derived sbuild chroots

[Moessbauer Felix]

On Mon, 2023-06-19 at 07:58 +0200, Jan Kiszka wrote:
> On 13.06.23 08:24, Uladzimir Bely wrote:
> > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-
> > users
> > wrote:
> > > This patch adds support to create derived sbuild chroots to
> > > speedup
> > > the
> > > build process. For packages that share a large set of common
> > > build
> > > dependencies, a derived sbuild chroot can be created to avoid the
> > > overhead of installing all base build-deps on each sbuild
> > > invocation.
> > > 
> > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > > ---
> > >  doc/user_manual.md                            | 27
> > > +++++++++++++++++++
> > >  meta/classes/crossvars.bbclass                | 11 +++++---
> > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > >  3 files changed, 41 insertions(+), 4 deletions(-)
> > 
> > We are going to merge this in near few days in spite of an open
> > question left in discussion. The reasons:
> > 
> > 1. It passes internal CI (dev/fast/full).
> > 
> > 2. It was useful in case of 'meta-iot2050' downstream for rewriting
> > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > going to
> > be deleted with "Imager schroot migration" patchset that is to be
> > merged soon too.
> 
> To track what I discussed with Baurzhan offlist: I would be good to
> have
> some QA check running when using a derived sbuild chroot that the
> package built has all extra packages as part of its Build-Depends so
> that we are not create again silently broken debianizations. Any
> ideas
> how to achieve that best, considering also implicit inclusions of the
> Build-Depends?

This is hard to achieve. The only way I could currently imagine is to
run the same package build in the standard chroot and check if it still
properly builds (by definition, the derived chroot has a superset of
the packages). For the created artifact, too-many packages in a build
chroot are not problematic. Given that, at least the created packages
are fine.

> 
> Furthermore, I had a small comment on patch 2.

I would rather postpone the "derive chroot name" aspect to later. It is
a minor thing and currently I lack the time to implement it. Feel free
to add a patch on top.

Felix

> 
> Jan
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Uladzimir Bely]

On Mon, 2023-06-19 at 21:10 +0800, Moessbauer Felix wrote:
> On Mon, 2023-06-19 at 07:58 +0200, Jan Kiszka wrote:
> > On 13.06.23 08:24, Uladzimir Bely wrote:
> > > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-
> > > users
> > > wrote:
> > > > This patch adds support to create derived sbuild chroots to
> > > > speedup
> > > > the
> > > > build process. For packages that share a large set of common
> > > > build
> > > > dependencies, a derived sbuild chroot can be created to avoid
> > > > the
> > > > overhead of installing all base build-deps on each sbuild
> > > > invocation.
> > > > 
> > > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > > > ---
> > > >  doc/user_manual.md                            | 27
> > > > +++++++++++++++++++
> > > >  meta/classes/crossvars.bbclass                | 11 +++++---
> > > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > > >  3 files changed, 41 insertions(+), 4 deletions(-)
> > > 
> > > We are going to merge this in near few days in spite of an open
> > > question left in discussion. The reasons:
> > > 
> > > 1. It passes internal CI (dev/fast/full).
> > > 
> > > 2. It was useful in case of 'meta-iot2050' downstream for
> > > rewriting
> > > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > > going to
> > > be deleted with "Imager schroot migration" patchset that is to be
> > > merged soon too.
> > 
> > To track what I discussed with Baurzhan offlist: I would be good to
> > have
> > some QA check running when using a derived sbuild chroot that the
> > package built has all extra packages as part of its Build-Depends
> > so
> > that we are not create again silently broken debianizations. Any
> > ideas
> > how to achieve that best, considering also implicit inclusions of
> > the
> > Build-Depends?
> 
> This is hard to achieve. The only way I could currently imagine is to
> run the same package build in the standard chroot and check if it
> still
> properly builds (by definition, the derived chroot has a superset of
> the packages). For the created artifact, too-many packages in a build
> chroot are not problematic. Given that, at least the created packages
> are fine.
> 

Could not it be some variable (disabled by default) that globally
prohibits changes of SBUILD_FLAVOR and SBUILD_CHROOT_PREINSTALL_COMMON
by packages/bbclasses?

It could be enabled in CI in order to build all packages with "clean"
schroots, but disabled by default to speedup development process.

> > 
> > Furthermore, I had a small comment on patch 2.
> 
> I would rather postpone the "derive chroot name" aspect to later. It
> is
> a minor thing and currently I lack the time to implement it. Feel
> free
> to add a patch on top.
> 
> Felix
> 
> > 
> > Jan
> > 
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Uladzimir Bely]

On Tue, 2023-06-20 at 09:25 +0300, Uladzimir Bely wrote:
> On Mon, 2023-06-19 at 21:10 +0800, Moessbauer Felix wrote:
> > On Mon, 2023-06-19 at 07:58 +0200, Jan Kiszka wrote:
> > > On 13.06.23 08:24, Uladzimir Bely wrote:
> > > > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-
> > > > users
> > > > wrote:
> > > > > This patch adds support to create derived sbuild chroots to
> > > > > speedup
> > > > > the
> > > > > build process. For packages that share a large set of common
> > > > > build
> > > > > dependencies, a derived sbuild chroot can be created to avoid
> > > > > the
> > > > > overhead of installing all base build-deps on each sbuild
> > > > > invocation.
> > > > > 
> > > > > Signed-off-by: Felix Moessbauer
> > > > > <felix.moessbauer@siemens.com>
> > > > > ---
> > > > >  doc/user_manual.md                            | 27
> > > > > +++++++++++++++++++
> > > > >  meta/classes/crossvars.bbclass                | 11 +++++---
> > > > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > > > >  3 files changed, 41 insertions(+), 4 deletions(-)
> > > > 
> > > > We are going to merge this in near few days in spite of an open
> > > > question left in discussion. The reasons:
> > > > 
> > > > 1. It passes internal CI (dev/fast/full).
> > > > 
> > > > 2. It was useful in case of 'meta-iot2050' downstream for
> > > > rewriting
> > > > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > > > going to
> > > > be deleted with "Imager schroot migration" patchset that is to
> > > > be
> > > > merged soon too.
> > > 
> > > To track what I discussed with Baurzhan offlist: I would be good
> > > to
> > > have
> > > some QA check running when using a derived sbuild chroot that the
> > > package built has all extra packages as part of its Build-Depends
> > > so
> > > that we are not create again silently broken debianizations. Any
> > > ideas
> > > how to achieve that best, considering also implicit inclusions of
> > > the
> > > Build-Depends?
> > 
> > This is hard to achieve. The only way I could currently imagine is
> > to
> > run the same package build in the standard chroot and check if it
> > still
> > properly builds (by definition, the derived chroot has a superset
> > of
> > the packages). For the created artifact, too-many packages in a
> > build
> > chroot are not problematic. Given that, at least the created
> > packages
> > are fine.
> > 
> 
> Could not it be some variable (disabled by default) that globally
> prohibits changes of SBUILD_FLAVOR and
> SBUILD_CHROOT_PREINSTALL_COMMON
> by packages/bbclasses?
> 
> It could be enabled in CI in order to build all packages with "clean"
> schroots, but disabled by default to speedup development process.
> 

I've experimented with such kind of "global disabling" derived schroots
and in general it works, but some moments are not still clear for me.

In the patch 2, we have SBUILD_FLAVOR="clang" set for `samefile`
package and that makes it use schroot with clang preinstalled. Also,
debian rules file is modified with `export CC=clang` in this case. As a
result, we have a package built with clang.

When I globally disable derived schroots, building `samefile` still
works since it is now built with gcc (debian rules are not modified).

I would expect a bit different behavior: debian rules files should not
be modified. Instead, `clang` is preinstalled as a dependency in
"clean" schroot and used as a compiler. So, in both cases (with
specific schroot and without it) the same compiler should be used - the
difference would be only in built time.

Probably, for the demo purposes it's OK, but I would expect that any
package is built with the same tools whether a derived schroot used or
not.

> > > 
> > > Furthermore, I had a small comment on patch 2.
> > 
> > I would rather postpone the "derive chroot name" aspect to later.
> > It
> > is
> > a minor thing and currently I lack the time to implement it. Feel
> > free
> > to add a patch on top.
> > 
> > Felix
> > 
> > > 
> > > Jan
> > > 
> > 
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Moessbauer Felix]

On Tue, 2023-06-20 at 11:05 +0300, Uladzimir Bely wrote:
> On Tue, 2023-06-20 at 09:25 +0300, Uladzimir Bely wrote:
> > On Mon, 2023-06-19 at 21:10 +0800, Moessbauer Felix wrote:
> > > On Mon, 2023-06-19 at 07:58 +0200, Jan Kiszka wrote:
> > > > On 13.06.23 08:24, Uladzimir Bely wrote:
> > > > > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via
> > > > > isar-
> > > > > users
> > > > > wrote:
> > > > > > This patch adds support to create derived sbuild chroots to
> > > > > > speedup
> > > > > > the
> > > > > > build process. For packages that share a large set of
> > > > > > common
> > > > > > build
> > > > > > dependencies, a derived sbuild chroot can be created to
> > > > > > avoid
> > > > > > the
> > > > > > overhead of installing all base build-deps on each sbuild
> > > > > > invocation.
> > > > > > 
> > > > > > Signed-off-by: Felix Moessbauer
> > > > > > <felix.moessbauer@siemens.com>
> > > > > > ---
> > > > > >  doc/user_manual.md                            | 27
> > > > > > +++++++++++++++++++
> > > > > >  meta/classes/crossvars.bbclass                | 11 +++++--
> > > > > > -
> > > > > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > > > > >  3 files changed, 41 insertions(+), 4 deletions(-)
> > > > > 
> > > > > We are going to merge this in near few days in spite of an
> > > > > open
> > > > > question left in discussion. The reasons:
> > > > > 
> > > > > 1. It passes internal CI (dev/fast/full).
> > > > > 
> > > > > 2. It was useful in case of 'meta-iot2050' downstream for
> > > > > rewriting
> > > > > "npm.bbclass" in order not to use buildchroot.bbclass (that
> > > > > is
> > > > > going to
> > > > > be deleted with "Imager schroot migration" patchset that is
> > > > > to
> > > > > be
> > > > > merged soon too.
> > > > 
> > > > To track what I discussed with Baurzhan offlist: I would be
> > > > good
> > > > to
> > > > have
> > > > some QA check running when using a derived sbuild chroot that
> > > > the
> > > > package built has all extra packages as part of its Build-
> > > > Depends
> > > > so
> > > > that we are not create again silently broken debianizations.
> > > > Any
> > > > ideas
> > > > how to achieve that best, considering also implicit inclusions
> > > > of
> > > > the
> > > > Build-Depends?
> > > 
> > > This is hard to achieve. The only way I could currently imagine
> > > is
> > > to
> > > run the same package build in the standard chroot and check if it
> > > still
> > > properly builds (by definition, the derived chroot has a superset
> > > of
> > > the packages). For the created artifact, too-many packages in a
> > > build
> > > chroot are not problematic. Given that, at least the created
> > > packages
> > > are fine.
> > > 
> > 
> > Could not it be some variable (disabled by default) that globally
> > prohibits changes of SBUILD_FLAVOR and
> > SBUILD_CHROOT_PREINSTALL_COMMON
> > by packages/bbclasses?
> > 
> > It could be enabled in CI in order to build all packages with
> > "clean"
> > schroots, but disabled by default to speedup development process.
> > 
> 
> I've experimented with such kind of "global disabling" derived
> schroots
> and in general it works, but some moments are not still clear for me.
> 
> In the patch 2, we have SBUILD_FLAVOR="clang" set for `samefile`
> package and that makes it use schroot with clang preinstalled. Also,
> debian rules file is modified with `export CC=clang` in this case. As
> a
> result, we have a package built with clang.

That's the idea. Compilers are never added explicitly as build
dependencies. We however have to enforce to use clang, as gcc is still
installed as the buildchroot is derived from the global one.

> 
> When I globally disable derived schroots, building `samefile` still
> works since it is now built with gcc (debian rules are not modified).
> 
> I would expect a bit different behavior: debian rules files should
> not
> be modified. Instead, `clang` is preinstalled as a dependency in
> "clean" schroot and used as a compiler. So, in both cases (with
> specific schroot and without it) the same compiler should be used -
> the
> difference would be only in built time.

I would expect, that this simply breaks. But it looks like an invalid
CC variable is simply ignored.

> 
> Probably, for the demo purposes it's OK, but I would expect that any
> package is built with the same tools whether a derived schroot used
> or
> not.

Agree. But this is just for demonstration. Feel free to remove this
test.

Felix

> 
> > > > 
> > > > Furthermore, I had a small comment on patch 2.
> > > 
> > > I would rather postpone the "derive chroot name" aspect to later.
> > > It
> > > is
> > > a minor thing and currently I lack the time to implement it. Feel
> > > free
> > > to add a patch on top.
> > > 
> > > Felix
> > > 
> > > > 
> > > > Jan
> > > > 
> > > 
> > 
> 

Re: [PATCH 2/2] add example how to use derived sbuild chroots

[Henning Schild]

Am Thu, 13 Apr 2023 07:00:26 +0000
schrieb "'Felix Moessbauer' via isar-users"
<isar-users@googlegroups.com>:

> This patch extends the samefile recipe to use a derived sbuild chroot
> to compile the tool using clang instead of gcc. While there are
> easier ways to achive this goal, it is still valuable to demonstrate
> how to use the infrastructure.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  meta-isar/recipes-app/samefile/samefile_2.14.bb       |  8 ++++++++
>  .../sbuild-chroot/sbuild-chroot-target-clang.bb       | 11
> +++++++++++ 2 files changed, 19 insertions(+)
>  create mode 100644
> meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> 
> diff --git a/meta-isar/recipes-app/samefile/samefile_2.14.bb
> b/meta-isar/recipes-app/samefile/samefile_2.14.bb index
> c53c9445..989e1983 100644 ---
> a/meta-isar/recipes-app/samefile/samefile_2.14.bb +++
> b/meta-isar/recipes-app/samefile/samefile_2.14.bb @@ -5,6 +5,9 @@
>  
>  inherit dpkg
>  
> +# for demo reasons, we compile with clang if not cross-compiling
> +SBUILD_FLAVOR = "${@ 'clang' if d.getVar('ISAR_CROSS_COMPILE') !=
> '1' else '' }" +
>  DEBIAN_DEPENDS = "\${misc:Depends}"
>  DESCRIPTION = "utility that finds files with identical contents"
>  
> @@ -42,4 +45,9 @@ EOF
>  
>      # We can also customize afterwards, in this case change the
> package section. sed -i -e 's/Section: misc/Section: utils/g'
> ${S}/debian/control +
> +    # use clang to compile (no cross support yet)
> +    if [ "${SBUILD_FLAVOR}" = "clang" ]; then
> +        sed -i -e 's/\(#\!.*\)/\1\nexport CC=clang/g'
> ${S}/debian/rules

This is going pretty far to demo the case. Even so far that
Build-Depends might be wrong and only satisfied with the magic.

You do not demo speed-ups by preinstalled build-deps, you demo hackery.

> +    fi
>  }
> diff --git
> a/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> new file mode 100644 index 00000000..3df16368 --- /dev/null
> +++
> b/meta-isar/recipes-devtools/sbuild-chroot/sbuild-chroot-target-clang.bb
> @@ -0,0 +1,11 @@ +# Root filesystem for packages building with clang
> +#
> +# This software is a part of ISAR.
> +# Copyright (C) 2023 Siemens AG
> +
> +DESCRIPTION = "Isar sbuild/schroot filesystem for target (clang
> variant)" +
> +require recipes-devtools/sbuild-chroot/sbuild-chroot-target.bb
> +
> +SBUILD_FLAVOR = "clang"
> +SBUILD_CHROOT_PREINSTALL_COMMON += "clang"

A bit unfortunate that the only one package name was also chosen as the
flavor name.

Maybe add some comments here, which one is a "name" and which one is a
"package list", or try to pick different strings as name.

Does that package benefit in any way from clang being preinstalled? I
guess only artificially because you patched in the CC=clang ...

Henning

Re: [PATCH 1/2] add support for derived sbuild chroots

[Henning Schild]

Am Mon, 19 Jun 2023 07:58:14 +0200
schrieb "'Jan Kiszka' via isar-users" <isar-users@googlegroups.com>:

> On 13.06.23 08:24, Uladzimir Bely wrote:
> > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-users
> > wrote:  
> >> This patch adds support to create derived sbuild chroots to speedup
> >> the
> >> build process. For packages that share a large set of common build
> >> dependencies, a derived sbuild chroot can be created to avoid the
> >> overhead of installing all base build-deps on each sbuild
> >> invocation.
> >>
> >> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> >> ---
> >>  doc/user_manual.md                            | 27
> >> +++++++++++++++++++
> >>  meta/classes/crossvars.bbclass                | 11 +++++---
> >>  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> >>  3 files changed, 41 insertions(+), 4 deletions(-)  
> > 
> > We are going to merge this in near few days in spite of an open
> > question left in discussion. The reasons:
> > 
> > 1. It passes internal CI (dev/fast/full).
> > 
> > 2. It was useful in case of 'meta-iot2050' downstream for rewriting
> > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > going to be deleted with "Imager schroot migration" patchset that
> > is to be merged soon too.  
> 
> To track what I discussed with Baurzhan offlist: I would be good to
> have some QA check running when using a derived sbuild chroot that the
> package built has all extra packages as part of its Build-Depends so
> that we are not create again silently broken debianizations. Any ideas
> how to achieve that best, considering also implicit inclusions of the
> Build-Depends?

As said by others that is kind of hard and the simplest would be a
double check in a "normal" sbuild where no extra packages are
installed. But i really do not see that as a big problem. Missing build
deps are unfortunate but not a big deal for anyone to resolve quickly
... should they ever surface because the build env for a given
code-base is ever brought on such a "minimal" build env.

In the past we shared a buildchroot and build order or parallel builds
of packages with conflicting build-depends would cause sporadic errors
that used to be hard to understand. Now nothing is shared ... if one
forgot a tool or library that is trivial to understand and fix.

IMHO not worth the effort. The feature is useful and the potential
problems rather harmless.

> Furthermore, I had a small comment on patch 2.

I kind of dislike that example but like the effort to put the new
feature under CI and present it with an example and test case to begin
with.

Henning
 
> Jan
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Moessbauer Felix]

On Wed, 2023-06-21 at 14:42 +0200, Henning Schild wrote:
> Am Mon, 19 Jun 2023 07:58:14 +0200
> schrieb "'Jan Kiszka' via isar-users" <isar-users@googlegroups.com>:
> 
> > On 13.06.23 08:24, Uladzimir Bely wrote:
> > > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-
> > > users
> > > wrote:  
> > > > This patch adds support to create derived sbuild chroots to
> > > > speedup
> > > > the
> > > > build process. For packages that share a large set of common
> > > > build
> > > > dependencies, a derived sbuild chroot can be created to avoid
> > > > the
> > > > overhead of installing all base build-deps on each sbuild
> > > > invocation.
> > > > 
> > > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > > > ---
> > > >  doc/user_manual.md                            | 27
> > > > +++++++++++++++++++
> > > >  meta/classes/crossvars.bbclass                | 11 +++++---
> > > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > > >  3 files changed, 41 insertions(+), 4 deletions(-)  
> > > 
> > > We are going to merge this in near few days in spite of an open
> > > question left in discussion. The reasons:
> > > 
> > > 1. It passes internal CI (dev/fast/full).
> > > 
> > > 2. It was useful in case of 'meta-iot2050' downstream for
> > > rewriting
> > > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > > going to be deleted with "Imager schroot migration" patchset that
> > > is to be merged soon too.  
> > 
> > To track what I discussed with Baurzhan offlist: I would be good to
> > have some QA check running when using a derived sbuild chroot that
> > the
> > package built has all extra packages as part of its Build-Depends
> > so
> > that we are not create again silently broken debianizations. Any
> > ideas
> > how to achieve that best, considering also implicit inclusions of
> > the
> > Build-Depends?
> 
> As said by others that is kind of hard and the simplest would be a
> double check in a "normal" sbuild where no extra packages are
> installed. But i really do not see that as a big problem. Missing
> build
> deps are unfortunate but not a big deal for anyone to resolve quickly
> ... should they ever surface because the build env for a given
> code-base is ever brought on such a "minimal" build env.
> 
> In the past we shared a buildchroot and build order or parallel
> builds
> of packages with conflicting build-depends would cause sporadic
> errors
> that used to be hard to understand. Now nothing is shared ... if one
> forgot a tool or library that is trivial to understand and fix.
> 
> IMHO not worth the effort. The feature is useful and the potential
> problems rather harmless.
> 
> > Furthermore, I had a small comment on patch 2.
> 
> I kind of dislike that example but like the effort to put the new
> feature under CI and present it with an example and test case to
> begin
> with.

How about merging just this patch, but not the example?
I currently cannot offer to provide a better example or CI integration.
Sorry.

If someone wants to implement it, feel free to send a patch.

Felix

> 
> Henning
>  
> > Jan
> > 
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Henning Schild]

Am Wed, 21 Jun 2023 21:22:30 +0800
schrieb Moessbauer Felix <felix.moessbauer@siemens.com>:

> On Wed, 2023-06-21 at 14:42 +0200, Henning Schild wrote:
> > Am Mon, 19 Jun 2023 07:58:14 +0200
> > schrieb "'Jan Kiszka' via isar-users" <isar-users@googlegroups.com>:
> >   
> > > On 13.06.23 08:24, Uladzimir Bely wrote:  
> > > > On Thu, 2023-04-13 at 07:00 +0000, 'Felix Moessbauer' via isar-
> > > > users
> > > > wrote:    
> > > > > This patch adds support to create derived sbuild chroots to
> > > > > speedup
> > > > > the
> > > > > build process. For packages that share a large set of common
> > > > > build
> > > > > dependencies, a derived sbuild chroot can be created to avoid
> > > > > the
> > > > > overhead of installing all base build-deps on each sbuild
> > > > > invocation.
> > > > > 
> > > > > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > > > > ---
> > > > >  doc/user_manual.md                            | 27
> > > > > +++++++++++++++++++
> > > > >  meta/classes/crossvars.bbclass                | 11 +++++---
> > > > >  .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
> > > > >  3 files changed, 41 insertions(+), 4 deletions(-)    
> > > > 
> > > > We are going to merge this in near few days in spite of an open
> > > > question left in discussion. The reasons:
> > > > 
> > > > 1. It passes internal CI (dev/fast/full).
> > > > 
> > > > 2. It was useful in case of 'meta-iot2050' downstream for
> > > > rewriting
> > > > "npm.bbclass" in order not to use buildchroot.bbclass (that is
> > > > going to be deleted with "Imager schroot migration" patchset
> > > > that is to be merged soon too.    
> > > 
> > > To track what I discussed with Baurzhan offlist: I would be good
> > > to have some QA check running when using a derived sbuild chroot
> > > that the
> > > package built has all extra packages as part of its Build-Depends
> > > so
> > > that we are not create again silently broken debianizations. Any
> > > ideas
> > > how to achieve that best, considering also implicit inclusions of
> > > the
> > > Build-Depends?  
> > 
> > As said by others that is kind of hard and the simplest would be a
> > double check in a "normal" sbuild where no extra packages are
> > installed. But i really do not see that as a big problem. Missing
> > build
> > deps are unfortunate but not a big deal for anyone to resolve
> > quickly ... should they ever surface because the build env for a
> > given code-base is ever brought on such a "minimal" build env.
> > 
> > In the past we shared a buildchroot and build order or parallel
> > builds
> > of packages with conflicting build-depends would cause sporadic
> > errors
> > that used to be hard to understand. Now nothing is shared ... if one
> > forgot a tool or library that is trivial to understand and fix.
> > 
> > IMHO not worth the effort. The feature is useful and the potential
> > problems rather harmless.
> >   
> > > Furthermore, I had a small comment on patch 2.  
> > 
> > I kind of dislike that example but like the effort to put the new
> > feature under CI and present it with an example and test case to
> > begin
> > with.  
> 
> How about merging just this patch, but not the example?
> I currently cannot offer to provide a better example or CI
> integration. Sorry.

I think that would even be acceptable. It is hard to demo really,
because if you do it to only one package it looks rather useless and
the demo does not make too much sense.

But in that case please send a v2 because you mention the samefile
example in the docs.

Henning

> If someone wants to implement it, feel free to send a patch.
> 
> Felix
> 
> > 
> > Henning
> >    
> > > Jan
> > >   
> >   
> 

Re: [PATCH 1/2] add support for derived sbuild chroots

[Henning Schild]

Am Thu, 13 Apr 2023 07:00:25 +0000
schrieb "'Felix Moessbauer' via isar-users"
<isar-users@googlegroups.com>:

> This patch adds support to create derived sbuild chroots to speedup
> the build process. For packages that share a large set of common build
> dependencies, a derived sbuild chroot can be created to avoid the
> overhead of installing all base build-deps on each sbuild invocation.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  doc/user_manual.md                            | 27
> +++++++++++++++++++ meta/classes/crossvars.bbclass                |
> 11 +++++--- .../sbuild-chroot/sbuild-chroot.inc           |  7 ++++-
>  3 files changed, 41 insertions(+), 4 deletions(-)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index 92075a88..e7a16e78 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -1348,3 +1348,30 @@ To enable it, add the below line to your
> local.conf file. ```
>  BASE_REPO_FEATURES = "cache-deb-src"
>  ```
> +
> +## Use a custom sbuild chroot to speedup build
> +
> +### Motivation
> +
> +There are use-cases, where many packages need to be compiled but all
> of them +need a similar base of build dependencies. In case the
> baseline is quite big, +this adds a significant overhead as the build
> dependencies are installed individually +for each and every package.
> +
> +### Solution
> +
> +By creating a dedicated sbuild chroot for this use-case, the
> baseline can be installed +first and then all package builds of this
> type can use it. For that, create a +new recipe with the name
> `sbuild-chroot-<host|target>-<flavor>`. In that recipe, +define the
> following: +
> +```
> +require recipes-devtools/sbuild-chroot/sbuild-chroot-<host|target>.bb
> +
> +SBUILD_FLAVOR = "<your flavor, e.g. clang>"
> +SBUILD_CHROOT_PREINSTALL_COMMON += "<base packages>"

in fact i would recommend a new variable here

SBUILD_CHROOT_PREINSTALL_EXTRA which defaults to ""

Henning

> +```
> +
> +Then, in the dpkg recipe of your package, simply set `SBUILD_FLAVOR
> = "<your flavor>"`. +To install additional packages into the sbuild
> chroot, add them to `SBUILD_CHROOT_PREINSTALL_COMMON`. +An full
> example is provided by the `samefile` recipe. diff --git
> a/meta/classes/crossvars.bbclass b/meta/classes/crossvars.bbclass
> index 201d460f..8d1da09c 100644 --- a/meta/classes/crossvars.bbclass
> +++ b/meta/classes/crossvars.bbclass
> @@ -10,22 +10,27 @@ python __anonymous() {
>      d.setVar('SCHROOT_USER', pwd.getpwuid(os.geteuid()).pw_name)
>  
>      mode = d.getVar('ISAR_CROSS_COMPILE')
> +
> +    # support derived schroots
> +    flavor = d.getVar('SBUILD_FLAVOR')
> +    flavor_suffix = ('-' + flavor) if flavor else ''
> +
>      distro_arch = d.getVar('DISTRO_ARCH')
>      if mode == "0" or d.getVar('HOST_ARCH') == distro_arch or
> distro_arch == None: d.setVar('BUILD_HOST_ARCH', distro_arch)
>          schroot_dir = d.getVar('SCHROOT_TARGET_DIR', False)
> -        sbuild_dep = "sbuild-chroot-target:do_build"
> +        sbuild_dep = "sbuild-chroot-target" + flavor_suffix +
> ":do_build" buildchroot_dir = d.getVar('BUILDCHROOT_TARGET_DIR',
> False) buildchroot_dep = "buildchroot-target:do_build"
>          sdk_toolchain = "build-essential"
>      else:
>          d.setVar('BUILD_HOST_ARCH', d.getVar('HOST_ARCH'))
>          schroot_dir = d.getVar('SCHROOT_HOST_DIR', False)
> -        sbuild_dep = "sbuild-chroot-host:do_build"
> +        sbuild_dep = "sbuild-chroot-host" + flavor_suffix +
> ":do_build" buildchroot_dir = d.getVar('BUILDCHROOT_HOST_DIR', False)
>          buildchroot_dep = "buildchroot-host:do_build"
>          sdk_toolchain = "crossbuild-essential-" + distro_arch
> -    d.setVar('SCHROOT_DIR', schroot_dir)
> +    d.setVar('SCHROOT_DIR', schroot_dir + flavor_suffix)
>      d.setVar('SCHROOT_DEP', sbuild_dep)
>      d.setVar('BUILDCHROOT_DIR', buildchroot_dir)
>      d.setVar('BUILDCHROOT_DEP', buildchroot_dep)
> diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
> b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc index
> fd8bb648..b786eef5 100644 ---
> a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc +++
> b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc @@ -11,6
> +11,11 @@ PV = "1.0" inherit rootfs
>  inherit compat
>  
> +# set the flavor to create derived sbuild chroots
> +# this maps to a schroot created by a recipe named
> sbuild-chroot-<host|target>-<flavor> +SBUILD_FLAVOR ??= ""
> +SBUILD_SCHROOT_SUFFIX = "${@ ('-' + d.getVar('SBUILD_FLAVOR')) if
> d.getVar('SBUILD_FLAVOR') else ''}" +
>  python() {
>      distro_gcc = d.getVar('DISTRO_GCC')
>      distro_arch = d.getVar('DISTRO_ARCH')
> @@ -52,7 +57,7 @@ ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}"
>  # We don't need /etc/apt/sources.list.d/isar-apt.list' while it's
> handled by sbuild ROOTFS_CONFIGURE_COMMAND:remove =
> "rootfs_configure_isar_apt" 
> -DEPLOY_SCHROOT = "${@d.getVar('SCHROOT_' +
> d.getVar('SBUILD_VARIANT').upper() + '_DIR')}" +DEPLOY_SCHROOT =
> "${@d.getVar('SCHROOT_' + d.getVar('SBUILD_VARIANT').upper() +
> '_DIR')}${SBUILD_SCHROOT_SUFFIX}" do_sbuildchroot_deploy[dirs] =
> "${DEPLOY_DIR}/schroot-${SBUILD_VARIANT}" do_sbuildchroot_deploy() {