[PATCH 1/3] testsuite: refactor

[Adriaan Schmidt <adriaan.schmidt@siemens.com>]

- make test initialization more explicit.
  tests now call init() to initialize build_dir and environment,
  and configure() to generate the config file and bitbake_args
- only configure() touches the config file. if config needs to change
  during one test case, it is created from scratch (no appending
  by the test case itself).
- build_dir is not passed via avocado parameter. each test
  can set it in the call to init(). this makes dependencies
  between tests explicit and permits parallelization.
- in the main invocation script, rename --build to --base.
  what we're setting here is not the bitbake build_dir but
  the base dir for avocado test output.

Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>
---
 scripts/ci_build.sh                |  22 +++---
 testsuite/build_test/build_test.py |  41 ++++++-----
 testsuite/build_test/cibase.py     |  72 ++++++-------------
 testsuite/build_test/cibuilder.py  | 109 +++++++++++++++++++++--------
 4 files changed, 134 insertions(+), 110 deletions(-)

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index e9ba034..339ebca 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -27,8 +27,8 @@ fi
 # Get Avocado build tests path
 BUILD_TEST_DIR="$(pwd)/testsuite/build_test"
 
-# Start build in Isar tree by default
-BUILD_DIR=./build
+# Start tests in current path by default
+BASE_DIR=./build
 
 # Check dependencies
 DEPENDENCIES="umoci skopeo"
@@ -45,8 +45,8 @@ show_help() {
     echo "    $0 [params]"
     echo
     echo "Parameters:"
-    echo "    -b, --build BUILD_DIR    set path to build directory. If not set,"
-    echo "                             the build will be started in current path."
+    echo "    -b, --base BASE_DIR      set path to base directory. If not set,"
+    echo "                             the tests will be started in current path."
     echo "    -c, --cross              enable cross-compilation."
     echo "    -d, --debug              enable debug bitbake output."
     echo "    -f, --fast               cross build reduced set of configurations."
@@ -73,8 +73,8 @@ do
         show_help
         exit 0
         ;;
-    -b|--build)
-        BUILD_DIR="$2"
+    -b|--base)
+        BASE_DIR="$2"
         shift
         ;;
     -c|--cross)
@@ -117,10 +117,10 @@ fi
 mkdir -p .config/avocado
 cat <<EOF > .config/avocado/avocado.conf
 [datadir.paths]
-base_dir = $(realpath $BUILD_DIR)/
-test_dir = $(realpath $BUILD_DIR)/tests
-data_dir = $(realpath $BUILD_DIR)/data
-logs_dir = $(realpath $BUILD_DIR)/job-results
+base_dir = $(realpath $BASE_DIR)/
+test_dir = $(realpath $BASE_DIR)/tests
+data_dir = $(realpath $BASE_DIR)/data
+logs_dir = $(realpath $BASE_DIR)/job-results
 EOF
 export VIRTUAL_ENV="./"
 
@@ -129,4 +129,4 @@ set -x
 
 avocado $VERBOSE run "$BUILD_TEST_DIR/build_test.py" \
     -t $TAGS --test-runner=runner --disable-sysinfo \
-    -p build_dir="$BUILD_DIR" -p quiet=$QUIET -p cross=$CROSS_BUILD
+    -p quiet=$QUIET -p cross=$CROSS_BUILD
diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
index 5a771d1..d343a78 100644
--- a/testsuite/build_test/build_test.py
+++ b/testsuite/build_test/build_test.py
@@ -30,7 +30,8 @@ class ReproTest(CIBaseTest):
             'mc:qemuarm64-stretch:isar-image-base'
                   ]
 
-        self.perform_repro_test(targets, 1)
+        self.init()
+        self.perform_repro_test(targets, signed=True)
 
     def test_repro_unsigned(self):
         targets = [
@@ -38,7 +39,8 @@ class ReproTest(CIBaseTest):
             'mc:qemuarm-stretch:isar-image-base'
                   ]
 
-        self.perform_repro_test(targets, 0)
+        self.init()
+        self.perform_repro_test(targets)
 
 class CcacheTest(CIBaseTest):
 
@@ -49,6 +51,7 @@ class CcacheTest(CIBaseTest):
     """
     def test_ccache_rebuild(self):
         targets = ['mc:de0-nano-soc-buster:isar-image-base']
+        self.init()
         self.perform_ccache_test(targets)
 
 class CrossTest(CIBaseTest):
@@ -69,15 +72,17 @@ class CrossTest(CIBaseTest):
             'mc:rpi-stretch:isar-image-base'
                   ]
 
-        self.perform_build_test(targets, 1, None)
+        self.init()
+        self.perform_build_test(targets, cross=True)
 
     def test_cross_ubuntu(self):
         targets = [
             'mc:qemuarm64-focal:isar-image-base'
                   ]
 
+        self.init()
         try:
-            self.perform_build_test(targets, 1, None)
+            self.perform_build_test(targets, cross=True)
         except:
             self.cancel('KFAIL')
 
@@ -86,8 +91,9 @@ class CrossTest(CIBaseTest):
             'mc:qemuarm-bullseye:isar-image-base'
                   ]
 
+        self.init()
         try:
-            self.perform_build_test(targets, 1, None)
+            self.perform_build_test(targets, cross=True)
         except:
             self.cancel('KFAIL')
 
@@ -101,7 +107,8 @@ class SdkTest(CIBaseTest):
     def test_sdk(self):
         targets = ['mc:qemuarm-stretch:isar-image-base']
 
-        self.perform_build_test(targets, 1, 'do_populate_sdk')
+        self.init()
+        self.perform_build_test(targets, bitbake_cmd='do_populate_sdk')
 
 class NoCrossTest(CIBaseTest):
 
@@ -130,11 +137,10 @@ class NoCrossTest(CIBaseTest):
             'mc:virtualbox-ova-buster:isar-image-base'
                   ]
 
+        self.init()
         # Cleanup after cross build
-        self.deletetmp(self.params.get('build_dir',
-                       default=os.path.dirname(__file__) + '/../../build'))
-
-        self.perform_build_test(targets, 0, None)
+        self.delete_from_build_dir('tmp')
+        self.perform_build_test(targets, cross=False)
 
     def test_nocross_bullseye(self):
         targets = [
@@ -145,8 +151,9 @@ class NoCrossTest(CIBaseTest):
             'mc:hikey-bullseye:isar-image-base'
                   ]
 
+        self.init()
         try:
-            self.perform_build_test(targets, 0, None)
+            self.perform_build_test(targets, cross=False)
         except:
             self.cancel('KFAIL')
 
@@ -158,8 +165,7 @@ class RebuildTest(CIBaseTest):
     :avocado: tags=rebuild,fast,full
     """
     def test_rebuild(self):
-        is_cross_build = int(self.params.get('cross', default=0))
-
+        self.init()
         layerdir_core = self.getlayerdir('core')
 
         dpkgbase_file = layerdir_core + '/classes/dpkg-base.bbclass'
@@ -169,8 +175,7 @@ class RebuildTest(CIBaseTest):
             file.write('do_fetch_append() {\n\n}')
 
         try:
-            self.perform_build_test('mc:qemuamd64-stretch:isar-image-base',
-                                    is_cross_build, None)
+            self.perform_build_test('mc:qemuamd64-stretch:isar-image-base')
         finally:
             self.restorefile(dpkgbase_file)
 
@@ -189,7 +194,8 @@ class ContainerImageTest(CIBaseTest):
             'mc:container-amd64-bullseye:isar-image-base'
                   ]
 
-        self.perform_container_test(targets, None)
+        self.init()
+        self.perform_build_test(targets, container=True)
 
 class ContainerSdkTest(CIBaseTest):
 
@@ -202,4 +208,5 @@ class ContainerSdkTest(CIBaseTest):
     def test_container_sdk(self):
         targets = ['mc:container-amd64-stretch:isar-image-base']
 
-        self.perform_container_test(targets, 'do_populate_sdk')
+        self.init()
+        self.perform_build_test(targets, bitbake_cmd='do_populate_sdk', container=True)
diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py
index 33bddfd..842ceaa 100644
--- a/testsuite/build_test/cibase.py
+++ b/testsuite/build_test/cibase.py
@@ -8,50 +8,21 @@ import time
 from cibuilder import CIBuilder
 from avocado.utils import process
 
-isar_root = os.path.dirname(__file__) + '/../..'
-
 class CIBaseTest(CIBuilder):
-
-    def prep(self, testname, targets, cross, debsrc_cache):
-        build_dir = self.params.get('build_dir', default=isar_root + '/build')
-        build_dir = os.path.realpath(build_dir)
-        quiet = int(self.params.get('quiet', default=0))
-        bitbake_args = '-v'
-
-        if quiet:
-            bitbake_args = ''
-
-        self.log.info('===================================================')
-        self.log.info('Running ' + testname + ' test for:')
-        self.log.info(targets)
-        self.log.info('Isar build folder is: ' + build_dir)
-        self.log.info('===================================================')
-
-        self.init(build_dir)
-        self.confprepare(build_dir, 1, cross, debsrc_cache)
-
-        return build_dir, bitbake_args;
-
-    def perform_build_test(self, targets, cross, bitbake_cmd):
-        build_dir, bb_args = self.prep('Isar build', targets, cross, 1)
+    def perform_build_test(self, targets, **kwargs):
+        self.configure(**kwargs)
 
         self.log.info('Starting build...')
 
-        self.bitbake(build_dir, targets, bitbake_cmd, bb_args)
-
-    def perform_repro_test(self, targets, signed):
-        cross = int(self.params.get('cross', default=0))
-        build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0)
+        self.bitbake(targets, **kwargs)
 
+    def perform_repro_test(self, targets, signed=False, **kwargs):
         gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key'
         gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key'
 
-        if signed:
-            with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-                # Enable use of signed cached base repository
-                file.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
+        self.configure(gpg_pub_key=gpg_pub_key if signed else None, **kwargs)
 
-        os.chdir(build_dir)
+        os.chdir(self.build_dir)
 
         os.environ['GNUPGHOME'] = tempfile.mkdtemp()
         result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key))
@@ -59,24 +30,22 @@ class CIBaseTest(CIBuilder):
         if result.exit_status:
             self.fail('GPG import failed')
 
-        self.bitbake(build_dir, targets, None, bb_args)
+        self.bitbake(targets, **kwargs)
 
-        self.deletetmp(build_dir)
-        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-            file.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
-            file.write('BB_NO_NETWORK = "1"\n')
+        self.delete_from_build_dir('tmp')
+        self.configure(gpg_pub_key=gpg_pub_key if signed else None, offline=True, **kwargs)
 
-        self.bitbake(build_dir, targets, None, bb_args)
+        self.bitbake(targets, **kwargs)
 
         # Disable use of cached base repository
-        self.confcleanup(build_dir)
+        self.unconfigure()
 
         if not signed:
             # Try to build with changed configuration with no cleanup
-            self.bitbake(build_dir, targets, None, bb_args)
+            self.bitbake(targets, **kwargs)
 
         # Cleanup
-        self.deletetmp(build_dir)
+        self.delete_from_build_dir('tmp')
 
     def perform_container_test(self, targets, bitbake_cmd):
         cross = int(self.params.get('cross', default=0))
@@ -93,21 +62,20 @@ class CIBaseTest(CIBuilder):
         self.deletetmp(build_dir)
         process.run('rm -rf ' + build_dir + '/ccache', sudo=True)
 
-        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-            file.write('USE_CCACHE = "1"\n')
-            file.write('CCACHE_TOP_DIR = "${TOPDIR}/ccache"')
+        self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('ccache')
 
         self.log.info('Starting build and filling ccache dir...')
         start = time.time()
-        self.bitbake(build_dir, targets, None, bb_args)
+        self.bitbake(targets, **kwargs)
         first_time = time.time() - start
         self.log.info('Non-cached build: ' + str(round(first_time)) + 's')
 
-        self.deletetmp(build_dir)
+        self.delete_from_build_dir('tmp')
 
         self.log.info('Starting build and using ccache dir...')
         start = time.time()
-        self.bitbake(build_dir, targets, None, bb_args)
+        self.bitbake(targets, **kwargs)
         second_time = time.time() - start
         self.log.info('Cached build: ' + str(round(second_time)) + 's')
 
@@ -116,5 +84,5 @@ class CIBaseTest(CIBuilder):
             self.fail('No speedup after rebuild with ccache')
 
         # Cleanup
-        self.deletetmp(build_dir)
-        process.run('rm -rf ' + build_dir + '/ccache', sudo=True)
+        self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('ccache')
diff --git a/testsuite/build_test/cibuilder.py b/testsuite/build_test/cibuilder.py
index 94786c7..2cfb6ec 100644
--- a/testsuite/build_test/cibuilder.py
+++ b/testsuite/build_test/cibuilder.py
@@ -2,7 +2,6 @@
 
 import logging
 import os
-import re
 import select
 import shutil
 import subprocess
@@ -11,7 +10,7 @@ from avocado import Test
 from avocado.utils import path
 from avocado.utils import process
 
-isar_root = os.path.dirname(__file__) + '/../..'
+isar_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 backup_prefix = '.ci-backup'
 
 app_log = logging.getLogger("avocado.app")
@@ -28,17 +27,60 @@ class CIBuilder(Test):
         self._file_handler.setFormatter(formatter)
         app_log.addHandler(self._file_handler)
 
-    def init(self, build_dir):
+    def init(self, build_dir='build'):
+        # initialize build_dir and setup environment
+        # needs to run once (per test case)
+        if hasattr(self, 'build_dir'):
+            self.error("Broken test implementation: init() called multiple times.")
+        self.build_dir = os.path.join(isar_root, build_dir)
         os.chdir(isar_root)
-        path.usable_rw_dir(build_dir)
+        path.usable_rw_dir(self.build_dir)
         output = process.getoutput('/bin/bash -c "source isar-init-build-env \
-                                    %s 2>&1 >/dev/null; env"' % build_dir)
+                                    %s 2>&1 >/dev/null; env"' % self.build_dir)
         env = dict(((x.split('=', 1) + [''])[:2] \
                     for x in output.splitlines() if x != ''))
         os.environ.update(env)
 
-    def confprepare(self, build_dir, compat_arch, cross, debsrc_cache):
-        with open(build_dir + '/conf/ci_build.conf', 'w') as f:
+    def check_init(self):
+        if not hasattr(self, 'build_dir'):
+            self.error("Broken test implementation: need to call init().")
+
+    def configure(self, compat_arch=True, cross=None, debsrc_cache=True,
+                  container=False, ccache=False, sstate=False, offline=False,
+                  gpg_pub_key=None, **kwargs):
+        # write configuration file and set bitbake_args
+        # can run multiple times per test case
+        self.check_init()
+
+        # get parameters from avocado cmdline
+        quiet = bool(int(self.params.get('quiet', default=0)))
+        if cross is None:
+            cross = bool(int(self.params.get('cross', default=0)))
+
+        # get parameters from environment
+        distro_apt_premir = os.getenv('DISTRO_APT_PREMIRRORS')
+
+        self.log.info(f'===================================================\n'
+                      f'Configuring build_dir {self.build_dir}\n'
+                      f'  compat_arch = {compat_arch}\n'
+                      f'  cross = {cross}\n'
+                      f'  debsrc_cache = {debsrc_cache}\n'
+                      f'  offline = {offline}\n'
+                      f'  container = {container}\n'
+                      f'  ccache = {ccache}\n'
+                      f'  sstate = {sstate}\n'
+                      f'  gpg_pub_key = {gpg_pub_key}\n'
+                      f'===================================================')
+
+        # determine bitbake_args
+        self.bitbake_args = []
+        if not quiet:
+            self.bitbake_args.append('-v')
+        if not sstate:
+            self.bitbake_args.append('--no-setscene')
+
+        # write ci_build.conf
+        with open(self.build_dir + '/conf/ci_build.conf', 'w') as f:
             if compat_arch:
                 f.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n')
                 f.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n')
@@ -48,36 +90,45 @@ class CIBuilder(Test):
                 f.write('ISAR_CROSS_COMPILE = "1"\n')
             if debsrc_cache:
                 f.write('BASE_REPO_FEATURES = "cache-deb-src"\n')
-            distro_apt_premir = os.getenv('DISTRO_APT_PREMIRRORS')
+            if offline:
+                f.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
+                f.write('BB_NO_NETWORK = "1"\n')
+            if container:
+                f.write('SDK_FORMATS = "docker-archive"\n')
+                f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n')
+            if gpg_pub_key:
+                f.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
             if distro_apt_premir:
                 f.write('DISTRO_APT_PREMIRRORS = "%s"\n' % distro_apt_premir)
 
-        with open(build_dir + '/conf/local.conf', 'r+') as f:
+        # include ci_build.conf in local.conf
+        with open(self.build_dir + '/conf/local.conf', 'r+') as f:
             for line in f:
                 if 'include ci_build.conf' in line:
                     break
             else:
                 f.write('\ninclude ci_build.conf')
 
-    def containerprep(self, build_dir):
-        with open(build_dir + '/conf/ci_build.conf', 'a') as f:
-            f.write('SDK_FORMATS = "docker-archive"\n')
-            f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n')
+    def unconfigure(self):
+        self.check_init()
+        open(self.build_dir + '/conf/ci_build.conf', 'w').close()
 
-    def confcleanup(self, build_dir):
-        open(build_dir + '/conf/ci_build.conf', 'w').close()
+    def delete_from_build_dir(self, path):
+        self.check_init()
+        process.run('rm -rf ' + self.build_dir + '/' + path, sudo=True)
 
-    def deletetmp(self, build_dir):
-        process.run('rm -rf ' + build_dir + '/tmp', sudo=True)
-
-    def bitbake(self, build_dir, target, cmd, args):
-        os.chdir(build_dir)
+    def bitbake(self, target, bitbake_cmd=None, **kwargs):
+        self.check_init()
+        self.log.info('===================================================')
+        self.log.info('Building ' + str(target))
+        self.log.info('===================================================')
+        os.chdir(self.build_dir)
         cmdline = ['bitbake']
-        if args:
-            cmdline.append(args)
-        if cmd:
+        if self.bitbake_args:
+            cmdline.extend(self.bitbake_args)
+        if bitbake_cmd:
             cmdline.append('-c')
-            cmdline.append(cmd)
+            cmdline.append(bitbake_cmd)
         if isinstance(target, list):
             cmdline.extend(target)
         else:
@@ -103,30 +154,28 @@ class CIBuilder(Test):
                 self.fail('Bitbake failed')
 
     def backupfile(self, path):
+        self.check_init()
         try:
             shutil.copy2(path, path + backup_prefix)
         except FileNotFoundError:
             self.log.warn(path + ' not exist')
 
     def backupmove(self, path):
+        self.check_init()
         try:
             shutil.move(path, path + backup_prefix)
         except FileNotFoundError:
             self.log.warn(path + ' not exist')
 
     def restorefile(self, path):
+        self.check_init()
         try:
             shutil.move(path + backup_prefix, path)
         except FileNotFoundError:
             self.log.warn(path + backup_prefix + ' not exist')
 
     def getlayerdir(self, layer):
-        try:
-            path.find_command('bitbake')
-        except path.CmdNotFoundError:
-            build_dir = self.params.get('build_dir',
-                                        default=isar_root + '/build')
-            self.init(build_dir)
+        self.check_init()
         output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="')
         env = dict(((x.split('=', 1) + [''])[:2] \
                     for x in output.splitlines() if x != ''))
-- 
2.30.2