Re: [PATCH 11/11] testsuite: Run custom commands on some qemu targets

[Henning Schild <henning.schild@siemens.com>]

Am Fri, 13 Jan 2023 08:19:42 +0100
schrieb Uladzimir Bely <ubely@ilbers.de>:

> For demonstaration purposes, build 'isar-image-ci' for some targets
> and add corresponding VM run tests with custom commands.
> 
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> ---
>  testsuite/citest.py | 70
> ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56
> insertions(+), 14 deletions(-)
> 
> diff --git a/testsuite/citest.py b/testsuite/citest.py
> index 08a2a00e..527e254f 100755
> --- a/testsuite/citest.py
> +++ b/testsuite/citest.py
> @@ -69,8 +69,8 @@ class CrossTest(CIBaseTest):
>      """
>      def test_cross(self):
>          targets = [
> -            'mc:qemuarm-buster:isar-image-base',
> -            'mc:qemuarm-bullseye:isar-image-base',
> +            'mc:qemuarm-buster:isar-image-ci',
> +            'mc:qemuarm-bullseye:isar-image-ci',
>              'mc:qemuarm64-bullseye:isar-image-base',
>              'mc:de0-nano-soc-bullseye:isar-image-base',
>              'mc:stm32mp15x-buster:isar-image-base'
> @@ -103,7 +103,7 @@ class CrossTest(CIBaseTest):
>  
>      def test_cross_bookworm(self):
>          targets = [
> -            'mc:qemuarm-bookworm:isar-image-base',
> +            'mc:qemuarm-bookworm:isar-image-ci',
>              'mc:qemuarm64-bookworm:isar-image-base'
>                    ]
>  
> @@ -158,9 +158,9 @@ class NoCrossTest(CIBaseTest):
>      """
>      def test_nocross(self):
>          targets = [
> -            'mc:qemuarm-buster:isar-image-base',
> +            'mc:qemuarm-buster:isar-image-ci',
>              'mc:qemuarm-bullseye:isar-image-base',
> -            'mc:qemuarm64-bullseye:isar-image-base',
> +            'mc:qemuarm64-bullseye:isar-image-ci',
>              'mc:qemui386-buster:isar-image-base',
>              'mc:qemui386-bullseye:isar-image-base',
>              'mc:qemuamd64-buster:isar-image-base',
> @@ -175,7 +175,7 @@ class NoCrossTest(CIBaseTest):
>              'mc:bananapi-bullseye:isar-image-base',
>              'mc:nanopi-neo-bullseye:isar-image-base',
>              'mc:stm32mp15x-bullseye:isar-image-base',
> -            'mc:qemuamd64-focal:isar-image-base'
> +            'mc:qemuamd64-focal:isar-image-ci'
>                    ]
>  
>          self.init()
> @@ -202,7 +202,7 @@ class NoCrossTest(CIBaseTest):
>              'mc:qemuamd64-bookworm:isar-image-base',
>              'mc:qemuarm-bookworm:isar-image-base',
>              'mc:qemui386-bookworm:isar-image-base',
> -            'mc:qemumipsel-bookworm:isar-image-base',
> +            'mc:qemumipsel-bookworm:isar-image-ci',
>              'mc:hikey-bookworm:isar-image-base'
>                    ]
>  
> @@ -301,11 +301,23 @@ class VmBootTestFast(CIBaseTest):
>      """
>      def test_arm_bullseye(self):
>          self.init()
> -        self.vm_start('arm','bullseye')
> +        self.vm_start('arm','bullseye', \
> +            image='isar-image-ci')
> +
> +    def test_arm_bullseye_example_module(self):
> +        self.init()
> +        self.vm_start('arm','bullseye', \
> +            image='isar-image-ci', cmd='lsmod | grep example_module')

I think when we add this, we should drop the log parsing on that one in
the same commit. In the end i would expect the only thing left to parse
will be that login prompt and we move all others to executing commands.

And again there is a weird mix of positional vs named args, like many
other lines in this commit.

>      def test_arm_buster(self):
>          self.init()
> -        self.vm_start('arm','buster')
> +        self.vm_start('arm','buster', \
> +            image='isar-image-ci')
> +
> +    def test_arm_buster_getty_target(self):
> +        self.init()
> +        self.vm_start('arm','buster', \
> +            image='isar-image-ci', cmd='systemctl is-active
> getty.target') 
>      def test_arm64_bullseye(self):
>          self.init()
> @@ -317,7 +329,13 @@ class VmBootTestFast(CIBaseTest):
>  
>      def test_arm_bookworm(self):
>          self.init()
> -        self.vm_start('arm','bookworm')
> +        self.vm_start('arm','bookworm', \
> +            image='isar-image-ci')
> +
> +    def test_arm_bookworm_example_module(self):
> +        self.init()
> +        self.vm_start('arm','bookworm', \
> +            image='isar-image-ci', cmd='lsmod | grep example_module')

All this is repeating over and over, maybe we can do better.

We could i.e. always run a default command after the parsing, that
could be "/bin/true". And some tests would set something more involved.

I also wonder how i would run multiple commands ... like a full "set
-e" shellscript without having to write several tests and booting many
times.

Maybe a 
 with start_vm as vm:
  run_cmd1(vm)
  run_cmd2(vm)

vm magically shuts down when leaving the "with"-scope

Henning

>  class VmBootTestFull(CIBaseTest):
>  
> @@ -332,11 +350,23 @@ class VmBootTestFull(CIBaseTest):
>  
>      def test_arm_buster(self):
>          self.init()
> -        self.vm_start('arm','buster')
> +        self.vm_start('arm','buster', \
> +            image='isar-image-ci')
> +
> +    def test_arm_buster_example_module(self):
> +        self.init()
> +        self.vm_start('arm','buster', \
> +            image='isar-image-ci', cmd='lsmod | grep example_module')
>  
>      def test_arm64_bullseye(self):
>          self.init()
> -        self.vm_start('arm64','bullseye')
> +        self.vm_start('arm64','bullseye', \
> +            image='isar-image-ci')
> +
> +    def test_arm64_bullseye_getty_target(self):
> +        self.init()
> +        self.vm_start('arm64','bullseye', \
> +            image='isar-image-ci', cmd='systemctl is-active
> getty.target') 
>      def test_amd64_bullseye(self):
>          self.init()
> @@ -355,7 +385,13 @@ class VmBootTestFull(CIBaseTest):
>  
>      def test_amd64_focal(self):
>          self.init()
> -        self.vm_start('amd64','focal')
> +        self.vm_start('amd64','focal', \
> +            image='isar-image-ci')
> +
> +    def test_amd64_focal_example_module(self):
> +        self.init()
> +        self.vm_start('amd64','focal', \
> +            image='isar-image-ci', cmd='lsmod | grep example_module')
>  
>      def test_amd64_bookworm(self):
>          self.init()
> @@ -371,4 +407,10 @@ class VmBootTestFull(CIBaseTest):
>  
>      def test_mipsel_bookworm(self):
>          self.init()
> -        self.vm_start('mipsel','bookworm')
> +        self.vm_start('mipsel','bookworm', \
> +            image='isar-image-ci')
> +
> +    def test_mipsel_bookworm_getty_target(self):
> +        self.init()
> +        self.vm_start('mipsel','bookworm', \
> +            image='isar-image-ci', cmd='systemctl is-active
> getty.target')