[RFC 1/1] mark ci tests that need attention

["'Felix Moessbauer' via isar-users" <isar-users@googlegroups.com>]

We mark the tests that are either suboptimal from a performance
PoV or that otherwise need attention. In general, the following things
need to be improved:

- use smaller baseline images for feature tests
- avoid repeated builds of kernels (especially with fast tag)

No functional change.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 testsuite/citest.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/testsuite/citest.py b/testsuite/citest.py
index 27cc9ff6..23cd06e9 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -199,11 +199,14 @@ class CrossTest(CIBaseTest):
         targets = [
             'mc:qemuarm-buster:isar-image-ci',
             'mc:qemuarm-bullseye:isar-image-ci',
+            # TODO: avoid kernel build (or at least build with ccache)
             'mc:de0-nano-soc-bullseye:isar-image-base',
+            # TODO: avoid kernel build
             'mc:stm32mp15x-bullseye:isar-image-base',
             'mc:qemuarm-bookworm:isar-image-ci',
             'mc:qemuarm64-focal:isar-image-base',
             'mc:nanopi-neo-efi-bookworm:isar-image-base',
+            # TODO: avoid kernel build
             'mc:phyboard-mira-bookworm:isar-image-base',
         ]
 
@@ -212,6 +215,7 @@ class CrossTest(CIBaseTest):
 
     def test_cross_debsrc(self):
         targets = [
+            # TODO: speedup task by removing prebuilt container added in 407a91322
             'mc:qemuarm64-bookworm:isar-image-ci',
         ]
 
@@ -220,6 +224,7 @@ class CrossTest(CIBaseTest):
 
     def test_cross_trixie(self):
         targets = [
+            # TODO: avoid sbuild-chroot-target-db2m
             'mc:qemuamd64-trixie:isar-image-base',
             'mc:qemuarm64-trixie:isar-image-base',
         ]
@@ -271,6 +276,9 @@ class KernelTests(CIBaseTest):
     """
     Tests associated with kernel builds and development.
     :avocado: tags=kernel,full
+
+    TODO: as we have dedicated kernel tests here, can we avoid the
+    kernel builds in test_cross?
     """
 
     def test_per_kernel(self):
@@ -306,6 +314,7 @@ class InitRdBaseTest(CIBaseTest):
     def deploy_dir_image(self, machine):
         return os.path.join(self.deploy_dir, 'images', machine)
 
+    # TODO: can we use a smaller baseline than isar-image-ci?
     def dracut_in_image(self, targets):
         machine = 'qemuamd64'
         distro = 'bookworm'
@@ -326,6 +335,7 @@ class InitRdBaseTest(CIBaseTest):
                 cmd="systemctl is-active dracut-shutdown"
             )
 
+    # TODO: use smaller baseline, currently we include containers
     def build_image_with_dependent_initrd(self, image, initrd,
                                           distro="debian-bookworm",
                                           machine="qemuamd64",
@@ -365,6 +375,7 @@ class InitRdTest(InitRdBaseTest):
 
     def test_dracut_in_image(self):
         """Test switch to dracut in an image recipe."""
+        # TODO: smaller baseline is sufficient to test this
         self.dracut_in_image(['mc:qemuamd64-bookworm:isar-image-ci'])
 
     def test_dracut_build_initrd(self):
@@ -377,6 +388,7 @@ class InitRdTest(InitRdBaseTest):
         lines = InitRdBaseTest.DRACUT_CONF
         lines.append("ROOTFS_INITRAMFS_GENERATOR_CMDLINE:append = ' --unknown-option'")
         self.init()
+        # TODO: smaller baseline is sufficient to test this
         self.perform_build_test('mc:qemuamd64-bookworm:isar-image-ci',
                                 should_fail=True, lines=lines)
 
@@ -390,6 +402,7 @@ class InitRdTest(InitRdBaseTest):
             f"INITRD_IMAGE = '{initrd}-{distro}-{machine}.initrd.img'",
             f"do_image[depends] += '{initrd}:do_build'"
         ]
+        # TODO: either use smaller baseline, or better a parsing-only test
 
         self.init()
         self.build_image_with_dependent_initrd('isar-image-ci', initrd,
@@ -400,6 +413,7 @@ class InitRdTest(InitRdBaseTest):
         initrd = 'isar-initramfs'
         lines = [f"IMAGE_INITRD = '{initrd}'"]
 
+        # TODO: use smaller baseline
         self.init()
         self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
 
@@ -424,6 +438,7 @@ class InitRdTest(InitRdBaseTest):
             f"IMAGE_INITRD = '{initrd}'",
             "INITRD_IMAGE = 'not-a-valid-initrd-file'"
         ]
+        # TODO: smaller baseline is sufficient to test this
         self.build_image_with_dependent_initrd('isar-image-ci', initrd, lines=lines)
 
 
@@ -435,6 +450,7 @@ class InitRdCrossTests(InitRdBaseTest):
 
     def test_dracut_in_image(self):
         """Test switch to dracut in an image recipe."""
+        # TODO: can be tested with bare-minimal images
         self.dracut_in_image([
             'mc:qemuamd64-bookworm:isar-image-ci',
             'mc:qemuamd64-trixie:isar-image-ci',
@@ -503,8 +519,11 @@ class NoCrossTest(CIBaseTest):
             'mc:qemuamd64-buster:isar-image-ci',
             'mc:qemuamd64-bullseye:isar-initramfs',
             'mc:qemumipsel-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:imx6-sabrelite-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:phyboard-mira-bullseye:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:hikey-bullseye:isar-image-base',
             'mc:virtualbox-bullseye:isar-image-base',
             'mc:virtualbox-bookworm:isar-image-base',
@@ -517,6 +536,7 @@ class NoCrossTest(CIBaseTest):
             'mc:qemuamd64-iso-bookworm:isar-image-ci',
             'mc:qemui386-bookworm:isar-image-base',
             'mc:qemumipsel-bookworm:isar-image-ci',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:hikey-bookworm:isar-image-base',
             'mc:beagleplay-bookworm:isar-image-base',
             'mc:qemuarm64-noble:isar-image-base',
@@ -532,6 +552,7 @@ class NoCrossTest(CIBaseTest):
     def test_nocross_debsrc(self):
         targets = [
             'mc:qemuarm-bookworm:isar-image-ci',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:stm32mp15x-bullseye:isar-image-base',
             'mc:de0-nano-soc-bookworm:isar-image-base',
         ]
@@ -565,6 +586,7 @@ class NoCrossTest(CIBaseTest):
             'mc:qemuarm64-trixie:isar-image-base',
             'mc:qemuarm-trixie:isar-image-base',
             'mc:qemuriscv64-trixie:isar-image-base',
+            # TODO: avoid targets with kernel builds or use ccache
             'mc:sifive-fu540-trixie:isar-image-base',
             'mc:starfive-visionfive2-trixie:isar-image-base',
         ]
@@ -598,6 +620,7 @@ class ContainerImageTest(CIBaseTest):
 
     @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
     def test_container_image(self):
+        # TODO: add trixie target
         targets = [
             'mc:container-amd64-buster:isar-image-base',
             'mc:container-amd64-bullseye:isar-image-base',
@@ -618,6 +641,7 @@ class ContainerSdkTest(CIBaseTest):
 
     @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
     def test_container_sdk(self):
+        # TODO: why is this only tested on bullseye? Consider moving to trixie
         targets = ['mc:container-amd64-bullseye:isar-image-base']
 
         self.init()
-- 
2.51.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20251212163448.2743851-1-felix.moessbauer%40siemens.com.