[PATCH v10 08/17] build_test: Refactoring build tests cases

[Anton Mikanovich <amikan@ilbers.de>]

Split build test into classes and separate cases. Implement test cases
tagging for external configuring. Also remove the usage of external
varianter plugin.
In this step, all ci_build test cases were copied as-is without
functional changes.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 testsuite/build_test/build_test.py | 208 ++++++++++++++++++++++++++---
 testsuite/build_test/cibase.py     | 110 +++++++++++++++
 testsuite/build_test/cibuilder.py  | 130 ++++++++++++++++++
 testsuite/build_test/run.sh        |   3 -
 testsuite/build_test/run_fast.sh   |   3 +
 testsuite/build_test/run_full.sh   |   3 +
 testsuite/build_test/variant.yaml  |  22 ---
 7 files changed, 434 insertions(+), 45 deletions(-)
 create mode 100644 testsuite/build_test/cibase.py
 create mode 100644 testsuite/build_test/cibuilder.py
 delete mode 100755 testsuite/build_test/run.sh
 create mode 100755 testsuite/build_test/run_fast.sh
 create mode 100755 testsuite/build_test/run_full.sh
 delete mode 100644 testsuite/build_test/variant.yaml

diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
index 7a55c2f..9038ec5 100644
--- a/testsuite/build_test/build_test.py
+++ b/testsuite/build_test/build_test.py
@@ -1,29 +1,197 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess32
-import sys
-from os.path import dirname
 
-from avocado import Test
+from avocado import skipUnless
+from avocado.utils import path
+from cibase import CIBaseTest
 
-class BuildTest(Test):
+UMOCI_AVAILABLE = True
+SKOPEO_AVAILABLE = True
+try:
+    path.find_command('umoci')
+except path.CmdNotFoundError:
+    UMOCI_AVAILABLE = False
+try:
+    path.find_command('skopeo')
+except path.CmdNotFoundError:
+    SKOPEO_AVAILABLE = False
 
-    def test(self):
-        # TODO: add default values
-        build_dir = self.params.get('build_dir', default='.')
-        arch = self.params.get('arch', default='arm')
-        distro = self.params.get('distro', default='stretch')
+class ReproTest(CIBaseTest):
 
-        self.log.info('===================================================')
-        self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
-        self.log.info('Isar build folder is: ' + build_dir)
-        self.log.info('===================================================')
+    """
+    Test cached base repository
 
-        #isar_root = dirname(__file__) + '/..'
-        os.chdir(build_dir)
-        cmdline = ['bitbake', 'mc:qemu' + arch + '-' + distro + ':isar-image-base']
-        p1 = subprocess32.run(cmdline)
+    :avocado: tags=repro,fast,full
+    """
+    def test_repro_signed(self):
+        targets = [
+            'mc:de0-nano-soc-buster:isar-image-base',
+            'mc:qemuarm64-stretch:isar-image-base'
+                  ]
 
-        if p1.returncode:
-            self.fail('Test failed')
+        self.perform_repro_test(targets, 1)
+
+    def test_repro_unsigned(self):
+        targets = [
+            'mc:qemuamd64-stretch:isar-image-base',
+            'mc:qemuarm-stretch:isar-image-base'
+                  ]
+
+        self.perform_repro_test(targets, 0)
+
+class CrossTest(CIBaseTest):
+
+    """
+    Start cross build for the defined set of configurations
+
+    :avocado: tags=cross,fast,full
+    """
+    def test_cross(self):
+        targets = [
+            'mc:qemuarm-stretch:isar-image-base',
+            'mc:qemuarm-buster:isar-image-base',
+            'mc:qemuarm64-stretch:isar-image-base',
+            'mc:qemuamd64-stretch:isar-image-base',
+            'mc:de0-nano-soc-buster:isar-image-base',
+            'mc:stm32mp15x-buster:isar-image-base',
+            'mc:rpi-stretch:isar-image-base',
+            'mc:qemuarm64-focal:isar-image-base'
+                  ]
+
+        self.perform_build_test(targets, 1, None)
+
+    def test_cross_bullseye(self):
+        targets = [
+            'mc:qemuarm-bullseye:isar-image-base'
+                  ]
+
+        try:
+            self.perform_build_test(targets, 1, None)
+        except:
+            self.cancel('KFAIL')
+
+class SdkTest(CIBaseTest):
+
+    """
+    In addition test SDK creation
+
+    :avocado: tags=sdk,fast,full
+    """
+    def test_sdk(self):
+        targets = ['mc:qemuarm-stretch:isar-image-base']
+
+        self.perform_build_test(targets, 1, 'do_populate_sdk')
+
+class NoCrossTest(CIBaseTest):
+
+    """
+    Start non-cross build for the defined set of configurations
+
+    :avocado: tags=nocross,full
+    """
+    def test_nocross(self):
+        targets = [
+            'mc:qemuarm-stretch:isar-image-base',
+            'mc:qemuarm-buster:isar-image-base',
+            'mc:qemuarm64-stretch:isar-image-base',
+            'mc:qemui386-stretch:isar-image-base',
+            'mc:qemui386-buster:isar-image-base',
+            'mc:qemuamd64-stretch:isar-image-base',
+            'mc:qemuamd64-buster:isar-image-base',
+            'mc:qemuamd64-buster-tgz:isar-image-base',
+            'mc:qemuamd64-buster:isar-initramfs',
+            'mc:qemumipsel-stretch:isar-image-base',
+            'mc:qemumipsel-buster:isar-image-base',
+            'mc:nand-ubi-demo-buster:isar-image-ubi',
+            'mc:rpi-stretch:isar-image-base',
+            'mc:qemuamd64-focal:isar-image-base'
+                  ]
+
+        # Cleanup after cross build
+        self.deletetmp(self.params.get('build_dir',
+                       default=os.path.dirname(__file__) + '/../../build'))
+
+        self.perform_build_test(targets, 0, None)
+
+    def test_nocross_bullseye(self):
+        targets = [
+            'mc:qemuamd64-bullseye:isar-image-base',
+            'mc:qemuarm-bullseye:isar-image-base',
+            'mc:qemui386-bullseye:isar-image-base',
+            'mc:qemumipsel-bullseye:isar-image-base'
+                  ]
+
+        try:
+            self.perform_build_test(targets, 0, None)
+        except:
+            self.cancel('KFAIL')
+
+class RebuildTest(CIBaseTest):
+
+    """
+    Test image rebuild
+
+    :avocado: tags=rebuild,fast,full
+    """
+    def test_rebuild(self):
+        is_cross_build = int(self.params.get('cross', default=0))
+
+        layerdir_core = self.getlayerdir('core')
+
+        dpkgbase_file = layerdir_core + '/classes/dpkg-base.bbclass'
+
+        self.backupfile(dpkgbase_file)
+        with open(dpkgbase_file, 'a') as file:
+            file.write('do_fetch_append() {\n\n}')
+
+        try:
+            self.perform_build_test('mc:qemuamd64-stretch:isar-image-base',
+                                    is_cross_build, None)
+        finally:
+            self.restorefile(dpkgbase_file)
+
+class WicTest(CIBaseTest):
+
+    """
+    Test wic --exclude-path
+
+    :avocado: tags=wic,fast,full
+    """
+    def test_wic_exclude(self):
+        # TODO: remove hardcoded filenames
+        wks_path = '/scripts/lib/wic/canned-wks/sdimage-efi.wks'
+        wic_path = '/tmp/deploy/images/qemuamd64/isar-image-base-debian-stretch-qemuamd64.wic.img'
+
+        self.perform_wic_test('mc:qemuamd64-stretch:isar-image-base',
+                              wks_path, wic_path)
+
+class ContainerImageTest(CIBaseTest):
+
+    """
+    Test containerized images creation
+
+    :avocado: tags=containerbuild,fast,full,container
+    """
+    @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
+    def test_nocross(self):
+        targets = [
+            'mc:container-amd64-stretch:isar-image-base',
+            'mc:container-amd64-buster:isar-image-base',
+            'mc:container-amd64-bullseye:isar-image-base'
+                  ]
+
+        self.perform_container_test(targets, None)
+
+class ContainerSdkTest(CIBaseTest):
+
+    """
+    Test SDK container image creation
+
+    :avocado: tags=containersdk,fast,full,container
+    """
+    @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found')
+    def test_container_sdk(self):
+        targets = ['mc:container-amd64-stretch:isar-image-base']
+
+        self.perform_container_test(targets, 'do_populate_sdk')
diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py
new file mode 100644
index 0000000..1a89f43
--- /dev/null
+++ b/testsuite/build_test/cibase.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import tempfile
+
+from cibuilder import CIBuilder
+from avocado.utils import process
+
+isar_root = os.path.dirname(__file__) + '/../..'
+
+class CIBaseTest(CIBuilder):
+
+    def prep(self, testname, targets, cross, debsrc_cache):
+        build_dir = self.params.get('build_dir', default=isar_root + '/build')
+        build_dir = os.path.realpath(build_dir)
+        quiet = int(self.params.get('quiet', default=0))
+        bitbake_args = '-v'
+
+        if quiet:
+            bitbake_args = ''
+
+        self.log.info('===================================================')
+        self.log.info('Running ' + testname + ' test for:')
+        self.log.info(targets)
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        self.init(build_dir)
+        self.confprepare(build_dir, 1, cross, debsrc_cache)
+
+        return build_dir, bitbake_args;
+
+    def perform_build_test(self, targets, cross, bitbake_cmd):
+        build_dir, bb_args = self.prep('Isar build', targets, cross, 1)
+
+        self.log.info('Starting build...')
+
+        self.bitbake(build_dir, targets, bitbake_cmd, bb_args)
+
+    def perform_repro_test(self, targets, signed):
+        cross = int(self.params.get('cross', default=0))
+        build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0)
+
+        gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key'
+        gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key'
+
+        if signed:
+            with open(build_dir + '/conf/ci_build.conf', 'a') as file:
+                # Enable use of signed cached base repository
+                file.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
+
+        os.chdir(build_dir)
+
+        os.environ['GNUPGHOME'] = tempfile.mkdtemp()
+        result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key))
+
+        if result.exit_status:
+            self.fail('GPG import failed')
+
+        self.bitbake(build_dir, targets, None, bb_args)
+
+        self.deletetmp(build_dir)
+        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
+            file.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
+            file.write('BB_NO_NETWORK = "1"\n')
+
+        self.bitbake(build_dir, targets, None, bb_args)
+
+        # Cleanup and disable use of signed cached base repository
+        self.deletetmp(build_dir)
+        self.confcleanup(build_dir)
+
+    def perform_wic_test(self, targets, wks_path, wic_path):
+        cross = int(self.params.get('cross', default=0))
+        build_dir, bb_args = self.prep('WIC exclude build', targets, cross, 1)
+
+        layerdir_isar = self.getlayerdir('isar')
+
+        wks_file = layerdir_isar + wks_path
+        wic_img = build_dir + wic_path
+
+        if not os.path.isfile(wic_img):
+            self.fail('No build started before: ' + wic_img + ' not exist')
+
+        self.backupfile(wks_file)
+        self.backupmove(wic_img)
+
+        with open(wks_file, 'r') as file:
+            lines = file.readlines()
+        with open(wks_file, 'w') as file:
+            for line in lines:
+                file.write(re.sub(r'part \/ ', 'part \/ --exclude-path usr ',
+                                  line))
+
+        try:
+            self.bitbake(build_dir, targets, None, bb_args)
+        finally:
+            self.restorefile(wks_file)
+
+        self.restorefile(wic_img)
+
+    def perform_container_test(self, targets, bitbake_cmd):
+        cross = int(self.params.get('cross', default=0))
+        build_dir, bb_args = self.prep('Isar Container', targets, cross, 1)
+
+        self.containerprep(build_dir)
+
+        self.bitbake(build_dir, targets, bitbake_cmd, bb_args)
+
diff --git a/testsuite/build_test/cibuilder.py b/testsuite/build_test/cibuilder.py
new file mode 100644
index 0000000..187363f
--- /dev/null
+++ b/testsuite/build_test/cibuilder.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+import logging
+import os
+import re
+import select
+import shutil
+import subprocess
+
+from avocado import Test
+from avocado.utils import path
+from avocado.utils import process
+
+isar_root = os.path.dirname(__file__) + '/../..'
+backup_prefix = '.ci-backup'
+
+app_log = logging.getLogger("avocado.app")
+
+class CIBuilder(Test):
+    def setUp(self):
+        super(CIBuilder, self).setUp()
+        job_log = os.path.join(os.path.dirname(self.logdir), '..', 'job.log')
+        self._file_handler = logging.FileHandler(filename=job_log)
+        self._file_handler.setLevel(logging.ERROR)
+        fmt = ('%(asctime)s %(module)-16.16s L%(lineno)-.4d %('
+               'levelname)-5.5s| %(message)s')
+        formatter = logging.Formatter(fmt=fmt)
+        self._file_handler.setFormatter(formatter)
+        app_log.addHandler(self._file_handler)
+
+    def init(self, build_dir):
+        os.chdir(isar_root)
+        path.usable_rw_dir(build_dir)
+        output = process.getoutput('/bin/bash -c "source isar-init-build-env \
+                                    %s 2>&1 >/dev/null; env"' % build_dir)
+        env = dict(((x.split('=', 1) + [''])[:2] \
+                    for x in output.splitlines() if x != ''))
+        os.environ.update(env)
+
+    def confprepare(self, build_dir, compat_arch, cross, debsrc_cache):
+        with open(build_dir + '/conf/ci_build.conf', 'w') as f:
+            if compat_arch:
+                f.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n')
+                f.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n')
+                f.write('ISAR_ENABLE_COMPAT_ARCH_debian-stretch_amd64 = "0"\n')
+                f.write('IMAGE_INSTALL += "kselftest"\n')
+            if cross:
+                f.write('ISAR_CROSS_COMPILE = "1"\n')
+            if debsrc_cache:
+                f.write('BASE_REPO_FEATURES = "cache-deb-src"\n')
+
+        with open(build_dir + '/conf/local.conf', 'r+') as f:
+            for line in f:
+                if 'include ci_build.conf' in line:
+                    break
+            else:
+                f.write('\ninclude ci_build.conf')
+
+    def containerprep(self, build_dir):
+        with open(build_dir + '/conf/ci_build.conf', 'a') as f:
+            f.write('SDK_FORMATS = "docker-archive"\n')
+            f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n')
+
+    def confcleanup(self, build_dir):
+        open(build_dir + '/conf/ci_build.conf', 'w').close()
+
+    def deletetmp(self, build_dir):
+        process.run('rm -rf ' + build_dir + '/tmp', sudo=True)
+
+    def bitbake(self, build_dir, target, cmd, args):
+        os.chdir(build_dir)
+        cmdline = ['bitbake']
+        if args:
+            cmdline.append(args)
+        if cmd:
+            cmdline.append('-c')
+            cmdline.append(cmd)
+        if isinstance(target, list):
+            cmdline.extend(target)
+        else:
+            cmdline.append(target)
+
+        with subprocess.Popen(" ".join(cmdline), stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE, universal_newlines=True,
+                              shell=True) as p1:
+            poller = select.poll()
+            poller.register(p1.stdout, select.POLLIN)
+            poller.register(p1.stderr, select.POLLIN)
+            while p1.poll() is None:
+                events = poller.poll(1000)
+                for fd, event in events:
+                    if fd == p1.stdout.fileno():
+                        self.log.info(p1.stdout.readline().rstrip())
+                    if fd == p1.stderr.fileno():
+                        app_log.error(p1.stderr.readline().rstrip())
+            p1.wait()
+            if p1.returncode:
+                self.fail('Bitbake failed')
+
+    def backupfile(self, path):
+        try:
+            shutil.copy2(path, path + backup_prefix)
+        except FileNotFoundError:
+            self.log.warn(path + ' not exist')
+
+    def backupmove(self, path):
+        try:
+            shutil.move(path, path + backup_prefix)
+        except FileNotFoundError:
+            self.log.warn(path + ' not exist')
+
+    def restorefile(self, path):
+        try:
+            shutil.move(path + backup_prefix, path)
+        except FileNotFoundError:
+            self.log.warn(path + backup_prefix + ' not exist')
+
+    def getlayerdir(self, layer):
+        try:
+            path.find_command('bitbake')
+        except path.CmdNotFoundError:
+            build_dir = self.params.get('build_dir',
+                                        default=isar_root + '/build')
+            self.init(build_dir)
+        output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="')
+        env = dict(((x.split('=', 1) + [''])[:2] \
+                    for x in output.splitlines() if x != ''))
+
+        return env['LAYERDIR_' + layer].strip('"')
+
diff --git a/testsuite/build_test/run.sh b/testsuite/build_test/run.sh
deleted file mode 100755
index a8ea9cc..0000000
--- a/testsuite/build_test/run.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR
diff --git a/testsuite/build_test/run_fast.sh b/testsuite/build_test/run_fast.sh
new file mode 100755
index 0000000..4d1bf44
--- /dev/null
+++ b/testsuite/build_test/run_fast.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py -t fast -p quiet=1 -p cross=1
diff --git a/testsuite/build_test/run_full.sh b/testsuite/build_test/run_full.sh
new file mode 100755
index 0000000..af5ec59
--- /dev/null
+++ b/testsuite/build_test/run_full.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py -t full -p quiet=1
diff --git a/testsuite/build_test/variant.yaml b/testsuite/build_test/variant.yaml
deleted file mode 100644
index 9ddc634..0000000
--- a/testsuite/build_test/variant.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-variants: !mux
-    stretch-amd64:
-        distro: "stretch"
-        arch:   "amd64"
-    stretch-i386:
-        distro: "stretch"
-        arch:   "i386"
-    stretch-arm:
-        distro: "stretch"
-        arch:   "arm"
-    stretch-arm64:
-        distro: "stretch"
-        arch:   "arm64"
-    buster-amd64:
-        distro: "buster"
-        arch:   "amd64"
-    buster-i386:
-        distro: "buster"
-        arch:   "i386"
-    buster-arm:
-        distro: "buster"
-        arch:   "arm"
-- 
2.25.1