Re: [PATCH v2] testsuite: Initial Avocado test implementation

[Claudius Heine <claudius.heine.ext@siemens.com>]

Hi,

On 03/04/2019 16.14, Baurzhan Ismagulov wrote:
> From: Alexander Smirnov <asmirnov@ilbers.de>
> 
> Currently, we have several test cases executed from shell scripts. As test
> coverage increases, adding test cases and reading the results becomes complex.
> Avocado testing framework provides the necessary infrastructure to organize
> test cases. This patch adds the initial Avocado test case support for future CI
> migration.
> 
> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> Tested-by: Maxim Yu. Osipov <mosipov@ilbers.de>
> ---
>   testsuite/README.md                    | 34 +++++++++++++
>   testsuite/build_test/build_test.py     | 29 +++++++++++
>   testsuite/build_test/run.sh            |  3 ++
>   testsuite/build_test/variant.yaml      | 22 +++++++++
>   testsuite/start_vm.py                  | 88 ++++++++++++++++++++++++++++++++++
>   testsuite/vm_boot_test/run.sh          |  3 ++
>   testsuite/vm_boot_test/variant.yaml    | 22 +++++++++
>   testsuite/vm_boot_test/vm_boot_test.py | 51 ++++++++++++++++++++
>   8 files changed, 252 insertions(+)
>   create mode 100644 testsuite/README.md
>   create mode 100644 testsuite/build_test/build_test.py
>   create mode 100755 testsuite/build_test/run.sh
>   create mode 100644 testsuite/build_test/variant.yaml
>   create mode 100755 testsuite/start_vm.py
>   create mode 100755 testsuite/vm_boot_test/run.sh
>   create mode 100644 testsuite/vm_boot_test/variant.yaml
>   create mode 100644 testsuite/vm_boot_test/vm_boot_test.py
> 
> diff --git a/testsuite/README.md b/testsuite/README.md
> new file mode 100644
> index 0000000..5e64223
> --- /dev/null
> +++ b/testsuite/README.md
> @@ -0,0 +1,34 @@
> +# Install Avocado
> +
> +The framework could be installed by using standard HOWTO:
> +
> +  https://github.com/avocado-framework/avocado#installing-with-standard-python-tools
> +
> +Then you need to install varianter yaml-to-mux plugin by following these instructions:
> +
> +  https://github.com/avocado-framework/avocado/tree/master/optional_plugins
> +
> +## For Debian 9.x
> +
> +        $ sudo apt-get install python-pip
> +        $ pip install --user subprocess32
> +        $ pip install --user avocado-framework
> +        $ pip install --user avocado-framework-plugin-varianter-yaml-to-mux
> +
> +# Pre
> +
> +        $ export PATH=$PATH:~/.local/bin
> +        $ cd isar
> +        $ source isar-init-build-env
> +
> +
> +# Run test
> +
> +Each testsuite directory contains:
> + - run.sh - script to start tests
> + - variants.yaml - set of input data
> + - *.py - test case
> +
> +# Other
> +
> +There is a tool start_vm.py which is the replacement for the bash script in isar/scripts directory.
> diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
> new file mode 100644
> index 0000000..4220d3a
> --- /dev/null
> +++ b/testsuite/build_test/build_test.py
> @@ -0,0 +1,29 @@
> +#!/usr/bin/env python3
> +
> +import os
> +import subprocess32
> +import sys
> +from os.path import dirname
> +
> +from avocado import Test
> +
> +class BuildTest(Test):
> +
> +    def test(self):
> +        # TODO: add default values
> +        build_dir = self.params.get('build_dir', default='.')
> +        arch = self.params.get('arch', default='arm')
> +        distro = self.params.get('distro', default='stretch')
> +
> +        self.log.info('===================================================')
> +        self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
> +        self.log.info('Isar build folder is: ' + build_dir)
> +        self.log.info('===================================================')
> +
> +        #isar_root = dirname(__file__) + '/..'
> +        os.chdir(build_dir)
> +        cmdline = ['bitbake', 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base']
> +        p1 = subprocess32.run(cmdline)
> +
> +        if p1.returncode:
> +            self.fail('Test failed')
> diff --git a/testsuite/build_test/run.sh b/testsuite/build_test/run.sh
> new file mode 100755
> index 0000000..a8ea9cc
> --- /dev/null
> +++ b/testsuite/build_test/run.sh
> @@ -0,0 +1,3 @@
> +#!/bin/bash
> +
> +avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR
> diff --git a/testsuite/build_test/variant.yaml b/testsuite/build_test/variant.yaml
> new file mode 100644
> index 0000000..9ddc634
> --- /dev/null
> +++ b/testsuite/build_test/variant.yaml
> @@ -0,0 +1,22 @@
> +variants: !mux
> +    stretch-amd64:
> +        distro: "stretch"
> +        arch:   "amd64"
> +    stretch-i386:
> +        distro: "stretch"
> +        arch:   "i386"
> +    stretch-arm:
> +        distro: "stretch"
> +        arch:   "arm"
> +    stretch-arm64:
> +        distro: "stretch"
> +        arch:   "arm64"
> +    buster-amd64:
> +        distro: "buster"
> +        arch:   "amd64"
> +    buster-i386:
> +        distro: "buster"
> +        arch:   "i386"
> +    buster-arm:
> +        distro: "buster"
> +        arch:   "arm"

What about the non-qemu variants? At least a build test would be nice.

Are you planning deploying and booting Isar with real HW in the CI as well?

Claudius

> diff --git a/testsuite/start_vm.py b/testsuite/start_vm.py
> new file mode 100755
> index 0000000..02a4b51
> --- /dev/null
> +++ b/testsuite/start_vm.py
> @@ -0,0 +1,88 @@
> +#!/usr/bin/env python3
> +#
> +# Helper script to start QEMU with Isar image
> +# Copyright (c) 2019, ilbers GmbH
> +
> +import argparse
> +import os
> +import subprocess
> +import sys
> +import time
> +
> +def get_bitbake_env(arch, distro):
> +    multiconfig = 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base'
> +    output = subprocess.check_output(['bitbake', '-e', str(multiconfig)])
> +    return output
> +
> +def get_bitbake_var(output, var):
> +    ret = ''
> +    for line in output.splitlines():
> +        if line.startswith(var):
> +            ret = line.split('"')[1]
> +    return ret
> +
> +def format_qemu_cmdline(arch, build, distro):
> +    bb_output = get_bitbake_env(arch, distro).decode()
> +
> +    rootfs_image = ''
> +    extra_args = ''
> +    cpu = ['']
> +
> +    image_type = get_bitbake_var(bb_output, 'IMAGE_TYPE')
> +    deploy_dir_image = get_bitbake_var(bb_output, 'DEPLOY_DIR_IMAGE')
> +    if image_type == 'ext4-img':
> +        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.ext4.img'
> +        kernel_image = deploy_dir_image + '/' + get_bitbake_var(bb_output, 'KERNEL_IMAGE')
> +        initrd_image = get_bitbake_var(bb_output, 'INITRD_IMAGE')
> +
> +        if not initrd_image:
> +            initrd_image = '/dev/null'
> +        else:
> +            initrd_image = deploy_dir_image + '/' + initrd_image
> +
> +        serial = get_bitbake_var(bb_output, 'MACHINE_SERIAL')
> +        root_dev = get_bitbake_var(bb_output, 'QEMU_ROOTFS_DEV')
> +        kargs = ['-append', '"console=' + serial + ' root=/dev/' + root_dev + ' rw"']
> +
> +        extra_args = ['-kernel', kernel_image, '-initrd', initrd_image]
> +        extra_args.extend(kargs)
> +    elif image_type == 'wic-img':
> +        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.wic.img'
> +        extra_args = ['-snapshot']
> +    else:
> +        raise ValueError('Invalid image type: ' + str(image_type))
> +
> +    qemu_arch = get_bitbake_var(bb_output, 'QEMU_ARCH')
> +    qemu_machine = get_bitbake_var(bb_output, 'QEMU_MACHINE')
> +    qemu_cpu = get_bitbake_var(bb_output, 'QEMU_CPU')
> +    qemu_disk_args = get_bitbake_var(bb_output, 'QEMU_DISK_ARGS')
> +
> +    qemu_disk_args = qemu_disk_args.replace('##ROOTFS_IMAGE##', deploy_dir_image + '/' + rootfs_image).split()
> +
> +    cmd = ['qemu-system-' + qemu_arch, '-m', '1024M']
> +
> +    if qemu_machine:
> +        cmd.extend(['-M', qemu_machine])
> +
> +    if qemu_cpu:
> +        cmd.extend(['-cpu', qemu_cpu])
> +
> +    cmd.extend(extra_args)
> +    cmd.extend(qemu_disk_args)
> +
> +    return cmd
> +
> +def start_qemu(arch, build, distro):
> +    cmdline = format_qemu_cmdline(arch, build, distro)
> +    cmdline.insert(1, '-nographic')
> +
> +    p1 = subprocess.call(cmdline)
> +
> +if __name__ == "__main__":
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument('-a', '--arch', choices=['arm', 'arm64', 'amd64', 'i386'], help='set isar machine architecture.', default='arm')
> +    parser.add_argument('-b', '--build', help='set path to build directory.', default=os.getcwd())
> +    parser.add_argument('-d', '--distro', choices=['jessie', 'stretch'], help='set isar Debian distribution.', default='stretch')
> +    args = parser.parse_args()
> +
> +    start_qemu(args.arch, args.build, args.distro)
> diff --git a/testsuite/vm_boot_test/run.sh b/testsuite/vm_boot_test/run.sh
> new file mode 100755
> index 0000000..9fdda95
> --- /dev/null
> +++ b/testsuite/vm_boot_test/run.sh
> @@ -0,0 +1,3 @@
> +#!/bin/bash
> +
> +avocado run vm_boot_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR time_to_wait:300
> diff --git a/testsuite/vm_boot_test/variant.yaml b/testsuite/vm_boot_test/variant.yaml
> new file mode 100644
> index 0000000..9ddc634
> --- /dev/null
> +++ b/testsuite/vm_boot_test/variant.yaml
> @@ -0,0 +1,22 @@
> +variants: !mux
> +    stretch-amd64:
> +        distro: "stretch"
> +        arch:   "amd64"
> +    stretch-i386:
> +        distro: "stretch"
> +        arch:   "i386"
> +    stretch-arm:
> +        distro: "stretch"
> +        arch:   "arm"
> +    stretch-arm64:
> +        distro: "stretch"
> +        arch:   "arm64"
> +    buster-amd64:
> +        distro: "buster"
> +        arch:   "amd64"
> +    buster-i386:
> +        distro: "buster"
> +        arch:   "i386"
> +    buster-arm:
> +        distro: "buster"
> +        arch:   "arm"
> diff --git a/testsuite/vm_boot_test/vm_boot_test.py b/testsuite/vm_boot_test/vm_boot_test.py
> new file mode 100644
> index 0000000..d4849c7
> --- /dev/null
> +++ b/testsuite/vm_boot_test/vm_boot_test.py
> @@ -0,0 +1,51 @@
> +#!/usr/bin/env python3
> +
> +import os
> +import subprocess32
> +import sys
> +import time
> +
> +from os.path import dirname
> +sys.path.append(dirname(__file__) + '/..')
> +
> +import start_vm
> +
> +from avocado import Test
> +
> +class VmBootTest(Test):
> +
> +    def test(self):
> +        # TODO: add default values
> +        build_dir = self.params.get('build_dir', default='.')
> +        arch = self.params.get('arch', default='arm')
> +        distro = self.params.get('distro', default='stretch')
> +        time_to_wait = self.params.get('time_to_wait', default=60)
> +
> +        self.log.info('===================================================')
> +        self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')')
> +        self.log.info('Isar build folder is: ' + build_dir)
> +        self.log.info('===================================================')
> +
> +        output_file = '/tmp/vm_boot_test.log'
> +        if os.path.exists(output_file):
> +            os.remove(output_file)
> +
> +        cmdline = start_vm.format_qemu_cmdline(arch, build_dir, distro)
> +        cmdline.insert(1, '-nographic')
> +        cmdline.append('-serial')
> +        cmdline.append('file:' + output_file)
> +
> +        self.log.info('QEMU boot line: ' + str(cmdline))
> +
> +        devnull = open(os.devnull, 'w')
> +
> +        p1 = subprocess32.Popen(cmdline, stdout=devnull, stderr=devnull)
> +        time.sleep(int(time_to_wait))
> +        p1.kill()
> +        p1.wait()
> +
> +        if os.path.exists(output_file):
> +            if 'isar login:' in open(output_file).read():
> +                return
> +
> +        self.fail('Test failed')
> 

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de