[PATCH] testsuite: Initial avocado tests implementation

["Maxim Yu. Osipov" <mosipov@ilbers.de>]

From: Alexander Smirnov <asmirnov@ilbers.de>

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
Tested-by: Maxim Yu. Osipov <mosipov@ilbers.de>
---
 testsuite/README.md                    | 34 +++++++++++++
 testsuite/build_test/build_test.py     | 29 +++++++++++
 testsuite/build_test/run.sh            |  3 ++
 testsuite/build_test/variant.yaml      | 22 +++++++++
 testsuite/start_vm.py                  | 88 ++++++++++++++++++++++++++++++++++
 testsuite/vm_boot_test/run.sh          |  3 ++
 testsuite/vm_boot_test/variant.yaml    | 22 +++++++++
 testsuite/vm_boot_test/vm_boot_test.py | 51 ++++++++++++++++++++
 8 files changed, 252 insertions(+)
 create mode 100644 testsuite/README.md
 create mode 100644 testsuite/build_test/build_test.py
 create mode 100755 testsuite/build_test/run.sh
 create mode 100644 testsuite/build_test/variant.yaml
 create mode 100755 testsuite/start_vm.py
 create mode 100755 testsuite/vm_boot_test/run.sh
 create mode 100644 testsuite/vm_boot_test/variant.yaml
 create mode 100644 testsuite/vm_boot_test/vm_boot_test.py

diff --git a/testsuite/README.md b/testsuite/README.md
new file mode 100644
index 0000000..5e64223
--- /dev/null
+++ b/testsuite/README.md
@@ -0,0 +1,34 @@
+# Install Avocado
+
+The framework could be installed by using standard HOWTO:
+
+  https://github.com/avocado-framework/avocado#installing-with-standard-python-tools
+
+Then you need to install varianter yaml-to-mux plugin by following these instructions:
+
+  https://github.com/avocado-framework/avocado/tree/master/optional_plugins
+
+## For Debian 9.x
+
+        $ sudo apt-get install python-pip
+        $ pip install --user subprocess32
+        $ pip install --user avocado-framework
+        $ pip install --user avocado-framework-plugin-varianter-yaml-to-mux
+
+# Pre
+
+        $ export PATH=$PATH:~/.local/bin
+        $ cd isar
+        $ source isar-init-build-env
+
+
+# Run test
+
+Each testsuite directory contains:
+ - run.sh - script to start tests
+ - variants.yaml - set of input data
+ - *.py - test case
+
+# Other
+
+There is a tool start_vm.py which is the replacement for the bash script in isar/scripts directory.
diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
new file mode 100644
index 0000000..4220d3a
--- /dev/null
+++ b/testsuite/build_test/build_test.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess32
+import sys
+from os.path import dirname
+
+from avocado import Test
+
+class BuildTest(Test):
+
+    def test(self):
+        # TODO: add default values
+        build_dir = self.params.get('build_dir', default='.')
+        arch = self.params.get('arch', default='arm')
+        distro = self.params.get('distro', default='stretch')
+
+        self.log.info('===================================================')
+        self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        #isar_root = dirname(__file__) + '/..'
+        os.chdir(build_dir)
+        cmdline = ['bitbake', 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base']
+        p1 = subprocess32.run(cmdline)
+
+        if p1.returncode:
+            self.fail('Test failed')
diff --git a/testsuite/build_test/run.sh b/testsuite/build_test/run.sh
new file mode 100755
index 0000000..a8ea9cc
--- /dev/null
+++ b/testsuite/build_test/run.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR
diff --git a/testsuite/build_test/variant.yaml b/testsuite/build_test/variant.yaml
new file mode 100644
index 0000000..9ddc634
--- /dev/null
+++ b/testsuite/build_test/variant.yaml
@@ -0,0 +1,22 @@
+variants: !mux
+    stretch-amd64:
+        distro: "stretch"
+        arch:   "amd64"
+    stretch-i386:
+        distro: "stretch"
+        arch:   "i386"
+    stretch-arm:
+        distro: "stretch"
+        arch:   "arm"
+    stretch-arm64:
+        distro: "stretch"
+        arch:   "arm64"
+    buster-amd64:
+        distro: "buster"
+        arch:   "amd64"
+    buster-i386:
+        distro: "buster"
+        arch:   "i386"
+    buster-arm:
+        distro: "buster"
+        arch:   "arm"
diff --git a/testsuite/start_vm.py b/testsuite/start_vm.py
new file mode 100755
index 0000000..02a4b51
--- /dev/null
+++ b/testsuite/start_vm.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+#
+# Helper script to start QEMU with Isar image
+# Copyright (c) 2019, ilbers GmbH
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+
+def get_bitbake_env(arch, distro):
+    multiconfig = 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base'
+    output = subprocess.check_output(['bitbake', '-e', str(multiconfig)])
+    return output
+
+def get_bitbake_var(output, var):
+    ret = ''
+    for line in output.splitlines():
+        if line.startswith(var):
+            ret = line.split('"')[1]
+    return ret
+
+def format_qemu_cmdline(arch, build, distro):
+    bb_output = get_bitbake_env(arch, distro).decode()
+
+    rootfs_image = ''
+    extra_args = ''
+    cpu = ['']
+
+    image_type = get_bitbake_var(bb_output, 'IMAGE_TYPE')
+    deploy_dir_image = get_bitbake_var(bb_output, 'DEPLOY_DIR_IMAGE')
+    if image_type == 'ext4-img':
+        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.ext4.img'
+        kernel_image = deploy_dir_image + '/' + get_bitbake_var(bb_output, 'KERNEL_IMAGE')
+        initrd_image = get_bitbake_var(bb_output, 'INITRD_IMAGE')
+
+        if not initrd_image:
+            initrd_image = '/dev/null'
+        else:
+            initrd_image = deploy_dir_image + '/' + initrd_image
+
+        serial = get_bitbake_var(bb_output, 'MACHINE_SERIAL')
+        root_dev = get_bitbake_var(bb_output, 'QEMU_ROOTFS_DEV')
+        kargs = ['-append', '"console=' + serial + ' root=/dev/' + root_dev + ' rw"']
+
+        extra_args = ['-kernel', kernel_image, '-initrd', initrd_image]
+        extra_args.extend(kargs)
+    elif image_type == 'wic-img':
+        rootfs_image = 'isar-image-base-debian-' + distro + '-qemu' + arch + '.wic.img'
+        extra_args = ['-snapshot']
+    else:
+        raise ValueError('Invalid image type: ' + str(image_type))
+
+    qemu_arch = get_bitbake_var(bb_output, 'QEMU_ARCH')
+    qemu_machine = get_bitbake_var(bb_output, 'QEMU_MACHINE')
+    qemu_cpu = get_bitbake_var(bb_output, 'QEMU_CPU')
+    qemu_disk_args = get_bitbake_var(bb_output, 'QEMU_DISK_ARGS')
+
+    qemu_disk_args = qemu_disk_args.replace('##ROOTFS_IMAGE##', deploy_dir_image + '/' + rootfs_image).split()
+
+    cmd = ['qemu-system-' + qemu_arch, '-m', '1024M']
+
+    if qemu_machine:
+        cmd.extend(['-M', qemu_machine])
+
+    if qemu_cpu:
+        cmd.extend(['-cpu', qemu_cpu])
+
+    cmd.extend(extra_args)
+    cmd.extend(qemu_disk_args)
+
+    return cmd
+
+def start_qemu(arch, build, distro):
+    cmdline = format_qemu_cmdline(arch, build, distro)
+    cmdline.insert(1, '-nographic')
+
+    p1 = subprocess.call(cmdline)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-a', '--arch', choices=['arm', 'arm64', 'amd64', 'i386'], help='set isar machine architecture.', default='arm')
+    parser.add_argument('-b', '--build', help='set path to build directory.', default=os.getcwd())
+    parser.add_argument('-d', '--distro', choices=['jessie', 'stretch'], help='set isar Debian distribution.', default='stretch')
+    args = parser.parse_args()
+
+    start_qemu(args.arch, args.build, args.distro)
diff --git a/testsuite/vm_boot_test/run.sh b/testsuite/vm_boot_test/run.sh
new file mode 100755
index 0000000..9fdda95
--- /dev/null
+++ b/testsuite/vm_boot_test/run.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run vm_boot_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR time_to_wait:300
diff --git a/testsuite/vm_boot_test/variant.yaml b/testsuite/vm_boot_test/variant.yaml
new file mode 100644
index 0000000..9ddc634
--- /dev/null
+++ b/testsuite/vm_boot_test/variant.yaml
@@ -0,0 +1,22 @@
+variants: !mux
+    stretch-amd64:
+        distro: "stretch"
+        arch:   "amd64"
+    stretch-i386:
+        distro: "stretch"
+        arch:   "i386"
+    stretch-arm:
+        distro: "stretch"
+        arch:   "arm"
+    stretch-arm64:
+        distro: "stretch"
+        arch:   "arm64"
+    buster-amd64:
+        distro: "buster"
+        arch:   "amd64"
+    buster-i386:
+        distro: "buster"
+        arch:   "i386"
+    buster-arm:
+        distro: "buster"
+        arch:   "arm"
diff --git a/testsuite/vm_boot_test/vm_boot_test.py b/testsuite/vm_boot_test/vm_boot_test.py
new file mode 100644
index 0000000..d4849c7
--- /dev/null
+++ b/testsuite/vm_boot_test/vm_boot_test.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess32
+import sys
+import time
+
+from os.path import dirname
+sys.path.append(dirname(__file__) + '/..')
+
+import start_vm
+
+from avocado import Test
+
+class VmBootTest(Test):
+
+    def test(self):
+        # TODO: add default values
+        build_dir = self.params.get('build_dir', default='.')
+        arch = self.params.get('arch', default='arm')
+        distro = self.params.get('distro', default='stretch')
+        time_to_wait = self.params.get('time_to_wait', default=60)
+
+        self.log.info('===================================================')
+        self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')')
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        output_file = '/tmp/vm_boot_test.log'
+        if os.path.exists(output_file):
+            os.remove(output_file)
+
+        cmdline = start_vm.format_qemu_cmdline(arch, build_dir, distro)
+        cmdline.insert(1, '-nographic')
+        cmdline.append('-serial')
+        cmdline.append('file:' + output_file)
+
+        self.log.info('QEMU boot line: ' + str(cmdline))
+
+        devnull = open(os.devnull, 'w')
+
+        p1 = subprocess32.Popen(cmdline, stdout=devnull, stderr=devnull)
+        time.sleep(int(time_to_wait))
+        p1.kill()
+        p1.wait()
+
+        if os.path.exists(output_file):
+            if 'isar login:' in open(output_file).read():
+                return
+
+        self.fail('Test failed')
-- 
2.11.0