[PATCH v3 07/11] build_test: Refactoring build tests cases

[Anton Mikanovich <amikan@ilbers.de>]

Split build test into classes and separate cases. Implement test cases
tagging for external configuring. Also remove the usage of external
varianter plugin.
In this step, all ci_build test cases were copied as-is without
functional changes.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 testsuite/build_test/build_test.py | 137 ++++++++++++++++++++++++-----
 testsuite/build_test/cibase.py     | 121 +++++++++++++++++++++++++
 testsuite/build_test/cibuilder.py  |  79 +++++++++++++++++
 testsuite/build_test/run.sh        |   3 -
 testsuite/build_test/run_fast.sh   |   3 +
 testsuite/build_test/run_full.sh   |   3 +
 testsuite/build_test/variant.yaml  |  22 -----
 7 files changed, 323 insertions(+), 45 deletions(-)
 create mode 100644 testsuite/build_test/cibase.py
 create mode 100644 testsuite/build_test/cibuilder.py
 delete mode 100755 testsuite/build_test/run.sh
 create mode 100755 testsuite/build_test/run_fast.sh
 create mode 100755 testsuite/build_test/run_full.sh
 delete mode 100644 testsuite/build_test/variant.yaml

diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
index 7a55c2f..bc33fce 100644
--- a/testsuite/build_test/build_test.py
+++ b/testsuite/build_test/build_test.py
@@ -1,29 +1,126 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess32
-import sys
-from os.path import dirname
 
-from avocado import Test
+from cibase import CIBaseTest
 
-class BuildTest(Test):
+class ReproTest(CIBaseTest):
 
-    def test(self):
-        # TODO: add default values
-        build_dir = self.params.get('build_dir', default='.')
-        arch = self.params.get('arch', default='arm')
-        distro = self.params.get('distro', default='stretch')
+    """
+    Test cached base repository
 
-        self.log.info('===================================================')
-        self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
-        self.log.info('Isar build folder is: ' + build_dir)
-        self.log.info('===================================================')
+    :avocado: tags=repro,fast,full
+    """
+    def test_repro(self):
+        targets_signed = ['mc:de0-nano-soc-buster:isar-image-base',
+                          'mc:qemuarm64-stretch:isar-image-base']
 
-        #isar_root = dirname(__file__) + '/..'
-        os.chdir(build_dir)
-        cmdline = ['bitbake', 'mc:qemu' + arch + '-' + distro + ':isar-image-base']
-        p1 = subprocess32.run(cmdline)
+        targets_unsigned = ['mc:qemuamd64-stretch:isar-image-base',
+                   'mc:qemuarm-stretch:isar-image-base']
+
+        is_cross_build = self.params.get('cross', default=0)
+
+        self.perform_repro_test(targets_signed, is_cross_build, 1)
+        self.perform_repro_test(targets_unsigned, is_cross_build, 0)
+
+class CrossTest(CIBaseTest):
+
+    """
+    Start cross build for the defined set of configurations
+
+    :avocado: tags=cross,fast,full
+    """
+    def test_cross(self):
+        targets = ['mc:qemuarm-stretch:isar-image-base',
+                   'mc:qemuarm-buster:isar-image-base',
+                   'mc:qemuarm64-stretch:isar-image-base',
+                   'mc:qemuamd64-stretch:isar-image-base',
+                   'mc:de0-nano-soc-buster:isar-image-base',
+                   'mc:stm32mp15x-buster:isar-image-base',
+                   'mc:rpi-stretch:isar-image-base',
+                   'mc:qemuarm64-focal:isar-image-base',
+                   'mc:qemuarm-bullseye:isar-image-base']
+
+        self.perform_build_test(targets, 1, None)
+
+class SdkTest(CIBaseTest):
+
+    """
+    In addition test SDK creation
+
+    :avocado: tags=sdk,fast,full
+    """
+    def test_sdk(self):
+        self.perform_build_test('mc:qemuarm-stretch:isar-image-base',
+                                1, 'do_populate_sdk')
+
+class NoCrossTest(CIBaseTest):
+
+    """
+    Start non-cross build for the defined set of configurations
+
+    :avocado: tags=nocross,full
+    """
+    def test_nocross(self):
+        targets = ['mc:qemuarm-stretch:isar-image-base',
+                   'mc:qemuarm-buster:isar-image-base',
+                   'mc:qemuarm64-stretch:isar-image-base',
+                   'mc:qemui386-stretch:isar-image-base',
+                   'mc:qemui386-buster:isar-image-base',
+                   'mc:qemuamd64-stretch:isar-image-base',
+                   'mc:qemuamd64-buster:isar-image-base',
+                   'mc:qemuamd64-buster-tgz:isar-image-base',
+                   'mc:qemuamd64-buster:isar-initramfs',
+                   'mc:qemumipsel-stretch:isar-image-base',
+                   'mc:qemumipsel-buster:isar-image-base',
+                   'mc:nand-ubi-demo-buster:isar-image-ubi',
+                   'mc:rpi-stretch:isar-image-base',
+                   'mc:qemuamd64-focal:isar-image-base',
+                   'mc:qemuamd64-bullseye:isar-image-base',
+                   'mc:qemuarm-bullseye:isar-image-base',
+                   'mc:qemui386-bullseye:isar-image-base',
+                   'mc:qemumipsel-bullseye:isar-image-base']
+
+        self.perform_build_test(targets, 0, None)
+
+class RebuildTest(CIBaseTest):
+
+    """
+    Test image rebuild
+
+    :avocado: tags=rebuild,fast,full
+    """
+    def test_rebuild(self):
+        is_cross_build = self.params.get('cross', default=0)
+
+        layerdir_core = self.getlayerdir('core')
+
+        dpkgbase_file = layerdir_core + '/classes/dpkg-base.bbclass'
+
+        self.backupfile(dpkgbase_file)
+        with open(dpkgbase_file, 'a') as file:
+            file.write('do_fetch_append() {\n\n}')
+
+        try:
+            self.perform_build_test('mc:qemuamd64-stretch:isar-image-base',
+                                    is_cross_build, None)
+        finally:
+            self.restorefile(dpkgbase_file)
+
+class WicTest(CIBaseTest):
+
+    """
+    Test wic --exclude-path
+
+    :avocado: tags=wic,fast,full
+    """
+    def test_wic_exclude(self):
+        is_cross_build = self.params.get('cross', default=0)
+
+        # TODO: remove hardcoded filenames
+        wks_path = '/scripts/lib/wic/canned-wks/sdimage-efi.wks'
+        wic_path = '/tmp/deploy/images/qemuamd64/isar-image-base-debian-stretch-qemuamd64.wic.img'
+
+        self.perform_wic_test('mc:qemuamd64-stretch:isar-image-base',
+                              is_cross_build, wks_path, wic_path)
 
-        if p1.returncode:
-            self.fail('Test failed')
diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py
new file mode 100644
index 0000000..68debca
--- /dev/null
+++ b/testsuite/build_test/cibase.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import tempfile
+
+from cibuilder import CIBuilder
+from avocado.utils import process
+
+isar_root = os.path.dirname(__file__) + '/../..'
+
+class CIBaseTest(CIBuilder):
+
+    def perform_build_test(self, targets, cross, bitbake_cmd):
+        build_dir = self.params.get('build_dir', default=isar_root + '/build')
+        quiet = self.params.get('quiet', default=None)
+        bitbake_args = '-v'
+
+        if quiet:
+            bitbake_args = ''
+
+        self.log.info('===================================================')
+        self.log.info('Running Isar build test for:')
+        self.log.info(targets)
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        self.init(build_dir)
+        self.confprepare(build_dir, 1, cross, 1)
+
+        self.log.info('Starting build...')
+
+        self.bitbake(build_dir, targets, bitbake_cmd, bitbake_args)
+
+    def perform_repro_test(self, targets, cross, signed):
+        build_dir = self.params.get('build_dir', default=isar_root + '/build')
+        quiet = self.params.get('quiet', default=None)
+        bitbake_args = '-v'
+
+        if quiet:
+            bitbake_args = ''
+
+        self.log.info('===================================================')
+        self.log.info('Running repro Isar build test for:')
+        self.log.info(targets)
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        self.init(build_dir)
+        self.confprepare(build_dir, 1, cross, 1)
+
+        gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key'
+        gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key'
+
+        if signed:
+            with open(build_dir + '/conf/ci_build.conf', 'a') as file:
+                # Enable use of signed cached base repository
+                file.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
+
+        os.chdir(build_dir)
+
+        os.environ['GNUPGHOME'] = tempfile.mkdtemp()
+        result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key))
+
+        if result.exit_status:
+            self.fail('GPG import failed')
+
+        self.bitbake(build_dir, targets, None, bitbake_args)
+
+        self.deletetmp(build_dir)
+        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
+            file.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
+            file.write('BB_NO_NETWORK = "1"\n')
+
+        self.bitbake(build_dir, targets, None, bitbake_args)
+
+        # Cleanup and disable use of signed cached base repository
+        self.deletetmp(build_dir)
+        self.confcleanup(build_dir)
+
+    def perform_wic_test(self, targets, cross, wks_path, wic_path):
+        build_dir = self.params.get('build_dir', default=isar_root + '/build')
+        quiet = self.params.get('quiet', default=None)
+        bitbake_args = '-v'
+
+        if quiet:
+            bitbake_args = ''
+
+        self.log.info('===================================================')
+        self.log.info('Running WIC exclude build test for:')
+        self.log.info(targets)
+        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('===================================================')
+
+        self.init(build_dir)
+        self.confprepare(build_dir, 1, cross, 1)
+
+        layerdir_isar = self.getlayerdir('isar')
+
+        wks_file = layerdir_isar + wks_path
+        wic_img = build_dir + wic_path
+
+        if not os.path.isfile(wic_img):
+            self.fail('Build was not performed before: ' + wic_img + ' not exist')
+
+        self.backupfile(wks_file)
+        self.backupmove(wic_img)
+
+        with open(wks_file, 'r') as file:
+            lines = file.readlines()
+        with open(wks_file, 'w') as file:
+            for line in lines:
+                file.write(re.sub(r'part \/ ', 'part \/ --exclude-path usr ', line))
+
+        try:
+            self.bitbake(build_dir, targets, None, bitbake_args)
+        finally:
+            self.restorefile(wks_file)
+
+        self.restorefile(wic_img)
+
diff --git a/testsuite/build_test/cibuilder.py b/testsuite/build_test/cibuilder.py
new file mode 100644
index 0000000..85a5678
--- /dev/null
+++ b/testsuite/build_test/cibuilder.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+
+from avocado import Test
+from avocado.utils import path
+from avocado.utils import process
+
+isar_root = os.path.dirname(__file__) + '/../..'
+
+class CIBuilder(Test):
+
+    def init(self, build_dir):
+        os.chdir(isar_root)
+        output = process.getoutput('/bin/bash -c "source isar-init-build-env \
+                                    %s 2>&1 >/dev/null; env"' % build_dir)
+        env = dict((line.split("=", 1) for line in output.splitlines()))
+        os.environ.update(env)
+
+    def confprepare(self, build_dir, compat_arch, cross, debsrc_cache):
+        with open(build_dir + '/conf/ci_build.conf', 'w') as file:
+            if compat_arch:
+                file.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n')
+                file.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n')
+                file.write('ISAR_ENABLE_COMPAT_ARCH_debian-stretch_amd64 = "0"\n')
+            if cross:
+                file.write('ISAR_CROSS_COMPILE = "1"\n')
+            if debsrc_cache:
+                file.write('BASE_REPO_FEATURES = "cache-deb-src"\n')
+
+        with open(build_dir + '/conf/local.conf', 'r+') as file:
+            for line in file:
+                if 'include ci_build.conf' in line:
+                    break
+            else:
+                file.write('\ninclude ci_build.conf')
+
+    def confcleanup(self, build_dir):
+        open(build_dir + '/conf/ci_build.conf', 'w').close()
+
+    def deletetmp(self, build_dir):
+        process.run('rm -rf ' + build_dir + '/tmp', sudo=True)
+
+    def bitbake(self, build_dir, target, cmd, args):
+        os.chdir(build_dir)
+        cmdline = ['bitbake']
+        if args:
+            cmdline.append(args)
+        if cmd:
+            cmdline.append('-c')
+            cmdline.append(cmd)
+        if isinstance(target, list):
+            cmdline.extend(target)
+        else:
+            cmdline.append(target)
+
+        process.run(" ".join(cmdline))
+
+    def backupfile(self, path):
+        shutil.copy2(path, path + '.ci-backup')
+
+    def backupmove(self, path):
+        shutil.move(path, path + '.ci-backup')
+
+    def restorefile(self, path):
+        shutil.move(path + '.ci-backup', path)
+
+    def getlayerdir(self, layer):
+        try:
+            path.find_command('bitbake')
+        except path.CmdNotFoundError:
+            build_dir = self.params.get('build_dir', default=isar_root + '/build')
+            self.init(build_dir)
+        output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="')
+        bitbake_env = dict((line.split("=", 1) for line in output.splitlines()))
+
+        return bitbake_env['LAYERDIR_' + layer].strip('"')
+
diff --git a/testsuite/build_test/run.sh b/testsuite/build_test/run.sh
deleted file mode 100755
index a8ea9cc..0000000
--- a/testsuite/build_test/run.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR
diff --git a/testsuite/build_test/run_fast.sh b/testsuite/build_test/run_fast.sh
new file mode 100755
index 0000000..4d1bf44
--- /dev/null
+++ b/testsuite/build_test/run_fast.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py -t fast -p quiet=1 -p cross=1
diff --git a/testsuite/build_test/run_full.sh b/testsuite/build_test/run_full.sh
new file mode 100755
index 0000000..af5ec59
--- /dev/null
+++ b/testsuite/build_test/run_full.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+avocado run build_test.py -t full -p quiet=1
diff --git a/testsuite/build_test/variant.yaml b/testsuite/build_test/variant.yaml
deleted file mode 100644
index 9ddc634..0000000
--- a/testsuite/build_test/variant.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-variants: !mux
-    stretch-amd64:
-        distro: "stretch"
-        arch:   "amd64"
-    stretch-i386:
-        distro: "stretch"
-        arch:   "i386"
-    stretch-arm:
-        distro: "stretch"
-        arch:   "arm"
-    stretch-arm64:
-        distro: "stretch"
-        arch:   "arm64"
-    buster-amd64:
-        distro: "buster"
-        arch:   "amd64"
-    buster-i386:
-        distro: "buster"
-        arch:   "i386"
-    buster-arm:
-        distro: "buster"
-        arch:   "arm"
-- 
2.25.1