Re: [PATCH v2 2/3] testsuite: speedup execution by limiting mc parsing to selected targets

["'MOESSBAUER, Felix' via isar-users" <isar-users@googlegroups.com>]

On Fri, 2026-07-24 at 16:56 +0200, Zhihang Wei wrote:
> On 7/20/26 09:37, 'Felix Moessbauer' via isar-users wrote:
> > The global isar mc.conf has over 70 multiconfig targets defined. This
> > makes initial parsing super slow, leading to parse times of 5-10 minutes
> > on testsuite tasks (per avocado task execution). To speed this up, We now
> > set the BBMULTICONFIG bitbake variable to exactly the targets we want to
> > test. This reduces the parsing time of tests to ~10s per test.
> > 
> > When running with sstate cache, this massively speeds up repeated runs,
> > as the parse time was the biggest single contributor.
> > 
> > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > ---
> >   testsuite/cibase.py           | 20 +++++++++++---------
> >   testsuite/cibuilder.py        | 17 +++++++++++++++++
> >   testsuite/repro-build-test.py |  1 +
> >   3 files changed, 29 insertions(+), 9 deletions(-)
> > 
> > diff --git a/testsuite/cibase.py b/testsuite/cibase.py
> > index 4a6308d0..52748004 100755
> > --- a/testsuite/cibase.py
> > +++ b/testsuite/cibase.py
> > @@ -15,7 +15,7 @@ from avocado.utils import process
> >   
> >   class CIBaseTest(CIBuilder):
> >       def perform_build_test(self, targets, should_fail=False, **kwargs):
> > -        self.configure(**kwargs)
> > +        self.configure(targets=targets, **kwargs)
> >   
> >           if bool(int(self.params.get('depgraph', default=0))):
> >               self.generate_dependency_graph(targets, reconfigure=False, **kwargs)
> > @@ -31,7 +31,7 @@ class CIBaseTest(CIBuilder):
> >                                     **kwargs):
> >           """Debug helper to better understand test task graphs."""
> >           if reconfigure:
> > -            self.configure(**kwargs)
> > +            self.configure(targets=targets, **kwargs)
> >   
> >           self.log.info("Generating dependency graph...")
> >   
> > @@ -41,7 +41,7 @@ class CIBaseTest(CIBuilder):
> >           self.move_in_build_dir('pn-buildlist', f"pn-buildlist-{self.name}")
> >   
> >       def perform_wic_partition_test(self, targets, wic_deploy_parts, **kwargs):
> > -        self.configure(wic_deploy_parts=wic_deploy_parts, **kwargs)
> > +        self.configure(targets=targets, wic_deploy_parts=wic_deploy_parts, **kwargs)
> >           self.bitbake(targets, **kwargs)
> >   
> >           wic_path = f"{self.build_dir}/tmp/deploy/images/*/*.wic.p1"
> > @@ -57,6 +57,7 @@ class CIBaseTest(CIBuilder):
> >           gpg_priv_key = os.path.join(keys_dir, 'test_priv.key')
> >   
> >           self.configure(
> > +            targets=targets,
> >               gpg_pub_key=gpg_pub_key if signed else None,
> >               sstate_dir='',
> >               **kwargs,
> > @@ -76,6 +77,7 @@ class CIBaseTest(CIBuilder):
> >               repro_type = 'signed' if signed else 'unsigned'
> >               self.move_in_build_dir('tmp', f"tmp_middle_repro_{repro_type}")
> >               self.configure(
> > +                targets=targets,
> >                   gpg_pub_key=gpg_pub_key if signed else None,
> >                   offline=True,
> >                   sstate_dir='',
> > @@ -89,7 +91,7 @@ class CIBaseTest(CIBuilder):
> >   
> >               if not signed:
> >                   # Try to build with changed configuration with no cleanup
> > -                self.configure(**kwargs)
> > +                self.configure(targets=targets, **kwargs)
> >                   self.bitbake(targets, **kwargs)
> >   
> >           finally:
> > @@ -109,7 +111,7 @@ class CIBaseTest(CIBuilder):
> >                               count += int(content[field])
> >               return count
> >   
> > -        self.configure(ccache=True, sstate_dir='', **kwargs)
> > +        self.configure(targets=targets, ccache=True, sstate_dir='', **kwargs)
> >   
> >           # Field that stores direct ccache hits
> >           direct_cache_hit = 22
> > @@ -173,7 +175,7 @@ class CIBaseTest(CIBuilder):
> >           process.run(f"git --work-tree={isar_sstate} checkout HEAD -- .")
> >   
> >           self.init('../build-sstate', isar_dir=isar_sstate)
> > -        self.configure(sstate=True, sstate_dir='', **kwargs)
> > +        self.configure(targets=image_target, sstate=True, sstate_dir='', **kwargs)
> >   
> >           # Cleanup sstate and tmp before test
> >           self.delete_from_build_dir('sstate-cache')
> > @@ -196,7 +198,7 @@ class CIBaseTest(CIBuilder):
> >           """
> >           Generate signature data for target(s) and check for cacheability issues
> >           """
> > -        self.configure(**kwargs)
> > +        self.configure(targets=targets, **kwargs)
> >           self.delete_from_build_dir('tmp_before_sstate')
> >           self.move_in_build_dir('tmp', 'tmp_before_sstate')
> >           self.bitbake(targets, sig_handler='none')
> > @@ -246,7 +248,7 @@ class CIBaseTest(CIBuilder):
> >                       return False
> >               return True
> >   
> > -        self.configure(sstate=True, sstate_dir='', **kwargs)
> > +        self.configure(targets=[image_target, package_target], sstate=True, sstate_dir='', **kwargs)
> >   
> >           deploy_dir = f"{self.build_dir}/tmp/deploy"
> >   
> > @@ -380,7 +382,7 @@ class CIBaseTest(CIBuilder):
> >                          sfiles[target][fname]  = CIUtils.get_tar_content(fname)
> >               return sfiles
> >   
> > -        self.configure(**kwargs)
> > +        self.configure(targets=targets, **kwargs)
> >   
> >           tmp_layer_dir = self.create_tmp_layer()
> >           try:
> > diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py
> > index d42e8631..1d415084 100755
> > --- a/testsuite/cibuilder.py
> > +++ b/testsuite/cibuilder.py
> > @@ -104,8 +104,21 @@ class CIBuilder(Test):
> >           if not hasattr(self, 'build_dir'):
> >               self.error("Broken test implementation: need to call init().")
> >   
> > +    @staticmethod
> > +    def _extract_multiconfigs(targets):
> > +        if isinstance(targets, str):
> > +            targets = [targets]
> > +        mc_names = set()
> > +        for target in targets:
> > +            if target.startswith('mc:'):
> > +                parts = target.split(':')
> > +                if len(parts) >= 2:
> > +                    mc_names.add(parts[1])
> > +        return mc_names
> > +
> >       def configure(
> >           self,
> > +        targets=None,
> >           compat_arch=True,
> >           cross=True,
> >           debsrc_cache=False,
> > @@ -196,6 +209,10 @@ class CIBuilder(Test):
> >   
> >           # write ci_build.conf
> >           with open(self.build_dir + '/conf/ci_build.conf', 'w') as f:
> > +            if targets:
> > +                mc_names = self._extract_multiconfigs(targets)
> > +                if mc_names:
> > +                    f.write('BBMULTICONFIG = "%s"\n\n' % ' '.join(sorted(mc_names)))
> >               if compat_arch:
> >                   f.write(
> >                       'ISAR_ENABLE_COMPAT_ARCH:amd64 = "1"\n'
> > diff --git a/testsuite/repro-build-test.py b/testsuite/repro-build-test.py
> > index c06269f5..c1b51aff 100755
> > --- a/testsuite/repro-build-test.py
> > +++ b/testsuite/repro-build-test.py
> > @@ -50,6 +50,7 @@ class ReproBuild(CIBuilder):
> >           # Build
> >           self.log.info("Started Build " + image_name)
> >           self.configure(
> > +            targets=target,
> >               source_date_epoch=source_date_epoch, use_apt_snapshot=True
> >           )
> >           self.bitbake(target)
> 
> Hi,
> 
> It seems this patch does not work with the installer testsuite (-t
>   installer). After the build testcase "test_installer_build" passed,
> the following "test_installer_run" and "test_installer_root_partition"
> tests failed.

I see. That's because we don't have the inner image in our
BBMULTICONFIG list anymore. The fix is trivial, just append the
following to test_installer_build:

lines=["BBMULTICONFIG:append = ' qemuamd64-bookworm'"]

Will send a v3 in a minute.

Felix

> 
> Some log:
> ......
> [stdlog] 2026-07-21 18:11:07,304 avocado.test stacktrace       L0049 
> ERROR|   File "/build/isar_wzh_devel_6a/76/bitbake/lib/bb/cooker.py", 
> line 1040, in findBestProvider
> [stdlog] 2026-07-21 18:11:07,304 avocado.test stacktrace       L0049 
> ERROR|     if pn in self.recipecaches[mc].providers:
> [stdlog] 2026-07-21 18:11:07,304 avocado.test stacktrace       L0049 
> ERROR|              ~~~~~~~~~~~~~~~~~^^^^
> [stdlog] 2026-07-21 18:11:07,304 avocado.test stacktrace       L0049 
> ERROR| KeyError: 'qemuamd64-bookworm'
> 
> Zhihang

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/4865535f9b767eb838afab41959755ced0fc56f0.camel%40siemens.com.