* [PATCH 1/2] testsuite: add support to run testsuite against specific SDE
2026-11-17 12:02 [PATCH 0/2] Isolate test classes, make them reproducible 'Felix Moessbauer' via isar-users
@ 2026-11-17 12:02 ` 'Felix Moessbauer' via isar-users
2026-11-17 12:03 ` [PATCH 2/2] testsuite: cleanup artifacts if not requested otherwise 'Felix Moessbauer' via isar-users
1 sibling, 0 replies; 3+ messages in thread
From: 'Felix Moessbauer' via isar-users @ 2026-11-17 12:02 UTC (permalink / raw)
To: isar-users; +Cc: Felix Moessbauer
The testsuite currently runs most tests against the upstream mirrors. By
that, it is executed against a moving baseline. This further is
problematic if an substantially older sstate cache is used during
bootstrapping, as then the referenced .debs are no longer available on
the mirrors.
We now add support to run the testsuite against a specific timestamp.
This can be used both for reproducing sporadic issues, as well as for
test speedup when running with an sstate cache. As some Ubuntu targets
do not have a working and stable snapshot mirror, we skip these in this
case.
Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
testsuite/README.md | 2 ++
testsuite/cibuilder.py | 7 +++++++
testsuite/citest.py | 33 ++++++++++++++++++++++++++++-----
3 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/testsuite/README.md b/testsuite/README.md
index bd9e998a..07bcfaf2 100644
--- a/testsuite/README.md
+++ b/testsuite/README.md
@@ -36,6 +36,8 @@ To generate a kas configuration for manual debugging, add `-p kasconfig=1`.
The configuration is written when the test starts to
`{build_dir}/kas-{testname}.yaml`.
+To build against a specific Debian snapshot date, add `-p sde=<SOURCE_DATE_EPOCH>`.
+
## Fast build test
```
diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py
index 0565f158..e810910a 100755
--- a/testsuite/cibuilder.py
+++ b/testsuite/cibuilder.py
@@ -55,6 +55,8 @@ class CanBeFinished(Exception):
class CIBuilder(Test):
def setUp(self):
super(CIBuilder, self).setUp()
+ self.sde = int(self.params.get('sde', default=0))
+ self.use_apt_snapshot = self.sde != 0
job_log = os.path.join(os.path.dirname(self.logdir), '..', 'job.log')
self._file_handler = logging.FileHandler(filename=job_log)
self._file_handler.setLevel(logging.ERROR)
@@ -156,6 +158,11 @@ class CIBuilder(Test):
if not rootless:
rootless = bool(int(self.params.get('rootless', default=0)))
+ if not source_date_epoch and self.sde:
+ source_date_epoch = self.sde
+ use_apt_snapshot = True
+ self.use_apt_snapshot = use_apt_snapshot
+
# set those to "" to not set dir value but use system default
if dl_dir is None:
dl_dir = os.getenv('DL_DIR')
diff --git a/testsuite/citest.py b/testsuite/citest.py
index 5918a5a4..db907e17 100644
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -348,10 +348,13 @@ class CrossTest(CIBaseTest):
'mc:qemuarm-bookworm:isar-image-ci',
'mc:qemuamd64-trixie:isar-image-base',
'mc:qemuarm64-trixie:isar-image-base',
- 'mc:qemuarm64-focal:isar-image-base',
'mc:nanopi-neo-efi-bookworm:isar-image-base',
'mc:qemuamd64-resolute:isar-image-base',
]
+ if not self.use_apt_snapshot:
+ targets += [
+ 'mc:qemuarm64-focal:isar-image-base',
+ ]
self.init()
self.perform_build_test(targets)
@@ -470,6 +473,9 @@ class CrossTest(CIBaseTest):
"""
:avocado: tags=startvm
"""
+ if self.use_apt_snapshot:
+ self.cancel('arm64 focal not supported with apt snapshot')
+
self.init()
self.vm_start('arm64', 'focal')
@@ -528,10 +534,13 @@ class CrossTest(CIBaseTest):
'mc:stm32mp15x-bullseye:kselftest',
'mc:qemuarm-bookworm:kselftest',
'mc:qemuarm64-bookworm:kselftest',
- 'mc:qemuarm64-focal:kselftest',
'mc:nanopi-neo-efi-trixie:kselftest',
'mc:phyboard-mira-trixie:kselftest',
]
+ if not self.use_apt_snapshot:
+ targets += [
+ 'mc:qemuarm64-focal:kselftest',
+ ]
self.init()
self.perform_build_test(targets)
@@ -596,8 +605,11 @@ class CrossTest(CIBaseTest):
"""
targets = [
'mc:qemuriscv64-trixie:isar-image-ci',
- 'mc:qemuriscv64-resolute:isar-image-ci',
]
+ if not self.use_apt_snapshot:
+ targets += [
+ 'mc:qemuriscv64-resolute:isar-image-ci',
+ ]
self.init()
try:
@@ -619,6 +631,8 @@ class CrossTest(CIBaseTest):
"""
:avocado: tags=startvm,riscv64
"""
+ if self.use_apt_snapshot:
+ self.cancel('riscv64 resolute not supported with apt snapshot')
self.init()
try:
self.vm_start('riscv64', 'resolute', image='isar-image-ci')
@@ -696,6 +710,8 @@ class UbuntuTest(CIBaseTest):
return machine in ('qemuamd64', 'qemuarm64')
def test_ubuntu_bootstrap_mirror(self):
+ if self.use_apt_snapshot:
+ self.cancel('snapshots enabled, but explicit no-snapshot test')
self.init()
self.perform_build_test(self.bootstrap_targets())
@@ -1105,12 +1121,15 @@ class NoCrossTest(CIBaseTest):
'mc:hikey-bookworm:isar-image-base',
'mc:hikey-trixie:isar-image-base',
'mc:beagleplay-bookworm:isar-image-base',
- 'mc:qemuarm64-noble:isar-image-base',
'mc:qemuamd64-noble:isar-image-base',
'mc:qemuamd64-jammy:isar-image-base',
- 'mc:qemuarm64-jammy:isar-image-base',
'mc:x86-pc-bookworm:isar-image-base',
]
+ if not self.use_apt_snapshot:
+ targets += [
+ 'mc:qemuarm64-noble:isar-image-base',
+ 'mc:qemuarm64-jammy:isar-image-base',
+ ]
self.init()
self.perform_build_test(targets, cross=False)
@@ -1333,6 +1352,8 @@ class NoCrossTest(CIBaseTest):
"""
:avocado: tags=startvm
"""
+ if self.use_apt_snapshot:
+ self.cancel('arm64 noble not supported with apt snapshot')
self.init()
self.vm_start('arm64', 'noble')
@@ -1354,6 +1375,8 @@ class NoCrossTest(CIBaseTest):
"""
:avocado: tags=startvm
"""
+ if self.use_apt_snapshot:
+ self.cancel('arm64 jammy not supported with apt snapshot')
self.init()
self.vm_start('arm64', 'jammy')
--
2.55.0
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20261117120300.374231-2-felix.moessbauer%40siemens.com.
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH 2/2] testsuite: cleanup artifacts if not requested otherwise
2026-11-17 12:02 [PATCH 0/2] Isolate test classes, make them reproducible 'Felix Moessbauer' via isar-users
2026-11-17 12:02 ` [PATCH 1/2] testsuite: add support to run testsuite against specific SDE 'Felix Moessbauer' via isar-users
@ 2026-11-17 12:03 ` 'Felix Moessbauer' via isar-users
1 sibling, 0 replies; 3+ messages in thread
From: 'Felix Moessbauer' via isar-users @ 2026-11-17 12:03 UTC (permalink / raw)
To: isar-users; +Cc: Felix Moessbauer
In CI runs we often see issues that only appear if executing tests
individually instead of all in a row. This happens, because subsequent
builds do not start in a fresh environment. To improve this, we now
clean the build/tmp* dirs after executing a test class. As avocado test
does not natively support this, we add the cleanup as a last test. Due
to our test-internal dependencies, we still cannot cleanup after each
testcase.
If the cleanup is not desired (e.g. for manual analysis), setting
-p cleanup=0 skips it.
Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
testsuite/README.md | 2 ++
testsuite/cibase.py | 18 ++++++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/testsuite/README.md b/testsuite/README.md
index 07bcfaf2..461e8c53 100644
--- a/testsuite/README.md
+++ b/testsuite/README.md
@@ -36,6 +36,8 @@ To generate a kas configuration for manual debugging, add `-p kasconfig=1`.
The configuration is written when the test starts to
`{build_dir}/kas-{testname}.yaml`.
+Artifacts are removed after each test class unless you pass `-p cleanup=0`.
+
To build against a specific Debian snapshot date, add `-p sde=<SOURCE_DATE_EPOCH>`.
## Fast build test
diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index 3f44e8ac..f824ae11 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -14,6 +14,24 @@ from avocado.utils import process
class CIBaseTest(CIBuilder):
+ def test_cleanup_artifacts(self):
+ """
+ Workaround for missing class teardown support: Cleanup
+ the build/tmp so that the next test class starts in
+ a clean environment. This test comes last, according
+ to the python class inheritance order.
+ """
+ if not bool(int(self.params.get('cleanup', default=1))):
+ return
+
+ self.init()
+ rootless = bool(int(self.params.get('rootless', default=0)))
+ clean_cmd = f"{isar_root}/scripts/isar-clean-builddir"
+ if rootless:
+ clean_cmd += " --rootless"
+ for tmp_dir in glob.glob(self.build_dir + '/' + 'tmp*'):
+ process.run(f"{clean_cmd} {tmp_dir}")
+
def perform_build_test(self, targets, should_fail=False, **kwargs):
self.configure(targets=targets, **kwargs)
--
2.55.0
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20261117120300.374231-3-felix.moessbauer%40siemens.com.
^ permalink raw reply [flat|nested] 3+ messages in thread