[RFC PATCH] testsuite: refactor

[Adriaan Schmidt <adriaan.schmidt@siemens.com>]

Hi Anton,

as discussed earlier, I played around with the build tests, and tried to
simplify them (especially the bitbake configuration with the growing number
of positional arguments).

The main changes I've made:

- make test initialization more explicit.
  tests now call init() to initialize build_dir and environment,
  and configure() to generate the config file and bitbake_args
- build_dir is not passed via avocado parameter. each test
  can set it in the call to init(). this makes dependencies
  between tests explicit and permits parallelization.
- only configure() touches the config file. if config needs to change
  during one test case, it is created from scratch (no appending
  by the test case itself).
- remove the perform_*_test functions and put their logic directly
  into the test cases. most of them were only used by one test case
  anyway.
- remove cibuilder.py and put its remaining logic (which is not much
  after the previous step) into cibase.py

With those changes I'm able to run the ci-fast tests. Haven't done
a thorough analysis yet, and I'm sure I've broken some details somewhere.
The work is not complete, but the general idea should be visible already.
Unfortunately the patch itself is not very readable, so best to look at the
updated files after applying.

Just let me know what you think and how you'd like to proceed.

Thanks and best regards,
Adriaan

Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>
---
 scripts/ci_build.sh                |  19 +-
 testsuite/build_test/build_test.py | 140 ++++++++++++--
 testsuite/build_test/cibase.py     | 286 ++++++++++++++++-------------
 testsuite/build_test/cibuilder.py  | 134 --------------
 4 files changed, 279 insertions(+), 300 deletions(-)
 delete mode 100644 testsuite/build_test/cibuilder.py

diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index e9ba034..34f68df 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -27,9 +27,6 @@ fi
 # Get Avocado build tests path
 BUILD_TEST_DIR="$(pwd)/testsuite/build_test"
 
-# Start build in Isar tree by default
-BUILD_DIR=./build
-
 # Check dependencies
 DEPENDENCIES="umoci skopeo"
 for prog in ${DEPENDENCIES} ; do
@@ -45,8 +42,6 @@ show_help() {
     echo "    $0 [params]"
     echo
     echo "Parameters:"
-    echo "    -b, --build BUILD_DIR    set path to build directory. If not set,"
-    echo "                             the build will be started in current path."
     echo "    -c, --cross              enable cross-compilation."
     echo "    -d, --debug              enable debug bitbake output."
     echo "    -f, --fast               cross build reduced set of configurations."
@@ -73,10 +68,6 @@ do
         show_help
         exit 0
         ;;
-    -b|--build)
-        BUILD_DIR="$2"
-        shift
-        ;;
     -c|--cross)
         CROSS_BUILD="1"
         ;;
@@ -117,10 +108,10 @@ fi
 mkdir -p .config/avocado
 cat <<EOF > .config/avocado/avocado.conf
 [datadir.paths]
-base_dir = $(realpath $BUILD_DIR)/
-test_dir = $(realpath $BUILD_DIR)/tests
-data_dir = $(realpath $BUILD_DIR)/data
-logs_dir = $(realpath $BUILD_DIR)/job-results
+base_dir = ./
+test_dir = ./tests
+data_dir = ./data
+logs_dir = ./job-results
 EOF
 export VIRTUAL_ENV="./"
 
@@ -129,4 +120,4 @@ set -x
 
 avocado $VERBOSE run "$BUILD_TEST_DIR/build_test.py" \
     -t $TAGS --test-runner=runner --disable-sysinfo \
-    -p build_dir="$BUILD_DIR" -p quiet=$QUIET -p cross=$CROSS_BUILD
+    -p quiet=$QUIET -p cross=$CROSS_BUILD
diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py
index de9e3fc..2643130 100644
--- a/testsuite/build_test/build_test.py
+++ b/testsuite/build_test/build_test.py
@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 
+import glob
 import os
+import re
+import tempfile
 
 from avocado import skipUnless
-from avocado.utils import path
+from avocado.utils import path, process
 from cibase import CIBaseTest
 
 UMOCI_AVAILABLE = True
@@ -30,7 +33,27 @@ class ReproTest(CIBaseTest):
             'mc:qemuarm64-stretch:isar-image-base'
                   ]
 
-        self.perform_repro_test(targets, 1)
+        gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key'
+        gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key'
+
+        self.init()
+        self.configure(gpg_pub_key=gpg_pub_key)
+
+        os.chdir(self.build_dir)
+        os.environ['GNUPGHOME'] = tempfile.mkdtemp()
+        result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key))
+
+        if result.exit_status:
+            self.fail('GPG import failed')
+
+        self.bitbake(targets)
+
+        self.delete_from_build_dir('tmp')
+        self.configure(offline=True, gpg_pub_key=gpg_pub_key)
+        self.bitbake(targets)
+
+        # Cleanup
+        self.delete_from_build_dir('tmp')
 
     def test_repro_unsigned(self):
         targets = [
@@ -38,7 +61,20 @@ class ReproTest(CIBaseTest):
             'mc:qemuarm-stretch:isar-image-base'
                   ]
 
-        self.perform_repro_test(targets, 0)
+        self.init()
+        self.configure()
+        self.bitbake(targets)
+
+        self.delete_from_build_dir('tmp')
+        self.configure(offline=True)
+        self.bitbake(targets)
+
+        # Disable use of cached base repository
+        self.unconfigure()
+        self.bitbake(targets)
+
+        # Cleanup
+        self.delete_from_build_dir('tmp')
 
 class CcacheTest(CIBaseTest):
 
@@ -49,7 +85,33 @@ class CcacheTest(CIBaseTest):
     """
     def test_ccache_rebuild(self):
         targets = ['mc:de0-nano-soc-buster:isar-image-base']
-        self.perform_ccache_test(targets)
+
+        self.init()
+        self.configure(ccache=True)
+        self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('ccache')
+
+        self.log.info('Starting build and filling ccache dir...')
+        start = time.time()
+        self.bitbake(targets)
+        first_time = time.time() - start
+        self.log.info('Non-cached build: ' + str(round(first_time)) + 's')
+
+        self.delete_from_build_dir('tmp')
+
+        self.log.info('Starting build and using ccache dir...')
+        start = time.time()
+        self.bitbake(targets)
+        second_time = time.time() - start
+        self.log.info('Cached build: ' + str(round(second_time)) + 's')
+
+        speedup_k = 1.1
+        if first_time / second_time < speedup_k:
+            self.fail('No speedup after rebuild with ccache')
+
+        # Cleanup
+        self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('ccache')
 
 class CrossTest(CIBaseTest):
 
@@ -69,15 +131,19 @@ class CrossTest(CIBaseTest):
             'mc:rpi-stretch:isar-image-base'
                   ]
 
-        self.perform_build_test(targets, 1, None)
+        self.init()
+        self.configure(cross=True)
+        self.bitbake(targets)
 
     def test_cross_ubuntu(self):
         targets = [
             'mc:qemuarm64-focal:isar-image-base'
                   ]
 
+        self.init()
+        self.configure(cross=True)
         try:
-            self.perform_build_test(targets, 1, None)
+            self.bitbake(targets)
         except:
             self.cancel('KFAIL')
 
@@ -86,8 +152,10 @@ class CrossTest(CIBaseTest):
             'mc:qemuarm-bullseye:isar-image-base'
                   ]
 
+        self.init()
+        self.configure(cross=True)
         try:
-            self.perform_build_test(targets, 1, None)
+            self.bitbake(targets)
         except:
             self.cancel('KFAIL')
 
@@ -101,7 +169,9 @@ class SdkTest(CIBaseTest):
     def test_sdk(self):
         targets = ['mc:qemuarm-stretch:isar-image-base']
 
-        self.perform_build_test(targets, 1, 'do_populate_sdk')
+        self.init()
+        self.configure(cross=True)
+        self.bitbake(targets, bitbake_cmd='do_populate_sdk')
 
 class NoCrossTest(CIBaseTest):
 
@@ -130,11 +200,12 @@ class NoCrossTest(CIBaseTest):
             'mc:virtualbox-ova-buster:isar-image-base'
                   ]
 
+        self.init()
+        self.configure(cross=False)
         # Cleanup after cross build
-        self.deletetmp(self.params.get('build_dir',
-                       default=os.path.dirname(__file__) + '/../../build'))
+        self.delete_from_build_dir('tmp')
 
-        self.perform_build_test(targets, 0, None)
+        self.bitbake(targets)
 
     def test_nocross_bullseye(self):
         targets = [
@@ -145,8 +216,10 @@ class NoCrossTest(CIBaseTest):
             'mc:hikey-bullseye:isar-image-base'
                   ]
 
+        self.init()
+        self.configure(cross=False)
         try:
-            self.perform_build_test(targets, 0, None)
+            self.bitbake(targets)
         except:
             self.cancel('KFAIL')
 
@@ -158,19 +231,18 @@ class RebuildTest(CIBaseTest):
     :avocado: tags=rebuild,fast,full
     """
     def test_rebuild(self):
-        is_cross_build = int(self.params.get('cross', default=0))
+        self.init()
+        self.configure()
 
         layerdir_core = self.getlayerdir('core')
 
         dpkgbase_file = layerdir_core + '/classes/dpkg-base.bbclass'
-
         self.backupfile(dpkgbase_file)
         with open(dpkgbase_file, 'a') as file:
             file.write('do_fetch_append() {\n\n}')
 
         try:
-            self.perform_build_test('mc:qemuamd64-stretch:isar-image-base',
-                                    is_cross_build, None)
+            self.bitbake('mc:qemuamd64-stretch:isar-image-base')
         finally:
             self.restorefile(dpkgbase_file)
 
@@ -186,8 +258,32 @@ class WicTest(CIBaseTest):
         wks_path = '/scripts/lib/wic/canned-wks/sdimage-efi.wks'
         wic_path = '/tmp/deploy/images/qemuamd64/isar-image-base-debian-stretch-qemuamd64.wic.img'
 
-        self.perform_wic_test('mc:qemuamd64-stretch:isar-image-base',
-                              wks_path, wic_path)
+        self.init()
+        self.configure()
+
+        layerdir_isar = self.getlayerdir('isar')
+        wks_file = layerdir_isar + wks_path
+        wic_img = self.build_dir + wic_path
+
+        if not os.path.isfile(wic_img):
+            self.fail('No build started before: ' + wic_img + ' not exist')
+
+        self.backupfile(wks_file)
+        self.backupmove(wic_img)
+        with open(wks_file, 'r') as file:
+            lines = file.readlines()
+        with open(wks_file, 'w') as file:
+            for line in lines:
+                file.write(re.sub(r'part \/ ', 'part \/ --exclude-path usr ',
+                                  line))
+
+        try:
+            self.bitbake('mc:qemuamd64-stretch:isar-image-base')
+        finally:
+            self.restorefile(wks_file)
+
+        self.restorefile(wic_img)
+
 
 class ContainerImageTest(CIBaseTest):
 
@@ -204,7 +300,9 @@ class ContainerImageTest(CIBaseTest):
             'mc:container-amd64-bullseye:isar-image-base'
                   ]
 
-        self.perform_container_test(targets, None)
+        self.init()
+        self.configure(container=True)
+        self.bitbake(targets)
 
 class ContainerSdkTest(CIBaseTest):
 
@@ -217,4 +315,6 @@ class ContainerSdkTest(CIBaseTest):
     def test_container_sdk(self):
         targets = ['mc:container-amd64-stretch:isar-image-base']
 
-        self.perform_container_test(targets, 'do_populate_sdk')
+        self.init()
+        self.configure(container=True)
+        self.bitbake(targets, bitbake_cmd='do_populate_sdk')
diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py
index 0b053aa..52b01ed 100644
--- a/testsuite/build_test/cibase.py
+++ b/testsuite/build_test/cibase.py
@@ -1,149 +1,171 @@
 #!/usr/bin/env python3
 
+import logging
 import os
 import re
-import tempfile
+import select
+import shutil
+import subprocess
 import time
 
-from cibuilder import CIBuilder
+from avocado import Test
+from avocado.utils import path
 from avocado.utils import process
 
-isar_root = os.path.dirname(__file__) + '/../..'
-
-class CIBaseTest(CIBuilder):
-
-    def prep(self, testname, targets, cross, debsrc_cache):
-        build_dir = self.params.get('build_dir', default=isar_root + '/build')
-        build_dir = os.path.realpath(build_dir)
-        quiet = int(self.params.get('quiet', default=0))
-        bitbake_args = '-v'
-
-        if quiet:
-            bitbake_args = ''
-
+isar_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+backup_prefix = '.ci-backup'
+app_log = logging.getLogger("avocado.app")
+
+class CIBaseTest(Test):
+    def setUp(self):
+        super(CIBaseTest, self).setUp()
+        job_log = os.path.join(os.path.dirname(self.logdir), '..', 'job.log')
+        self._file_handler = logging.FileHandler(filename=job_log)
+        self._file_handler.setLevel(logging.ERROR)
+        fmt = ('%(asctime)s %(module)-16.16s L%(lineno)-.4d %('
+               'levelname)-5.5s| %(message)s')
+        formatter = logging.Formatter(fmt=fmt)
+        self._file_handler.setFormatter(formatter)
+        app_log.addHandler(self._file_handler)
+
+
+    def init(self, build_dir='build'):
+        # initialize build_dir and setup environment
+        # needs to run once (per test case)
+        self.build_dir = os.path.join(isar_root, build_dir)
+        os.chdir(isar_root)
+        path.usable_rw_dir(self.build_dir)
+        output = process.getoutput('/bin/bash -c "source isar-init-build-env \
+                                    %s 2>&1 >/dev/null; env"' % self.build_dir)
+        env = dict(((x.split('=', 1) + [''])[:2] \
+                    for x in output.splitlines() if x != ''))
+        os.environ.update(env)
+
+
+    def configure(self, compat_arch=True, cross=None, debsrc_cache=True,
+                  container=False, ccache=False, offline=False,
+                  gpg_pub_key=None):
+        # write configuration file and set bitbake_args
+        # can run multiple times per test case
+
+        # get parameters from avocado cmdline
+        quiet = bool(int(self.params.get('quiet', default=0)))
+        if cross is None:
+            cross = bool(int(self.params.get('cross', default=0)))
+
+        # get parameters from environment
+        distro_apt_premir = os.getenv('DISTRO_APT_PREMIRRORS')
+
+        self.log.info(f'===================================================\n'
+                      f'Configuring build_dir {self.build_dir}\n'
+                      f'  compat_arch = {compat_arch}\n'
+                      f'  cross = {cross}\n'
+                      f'  debsrc_cache = {debsrc_cache}\n'
+                      f'  offline = {offline}\n'
+                      f'  container = {container}\n'
+                      f'  ccache = {ccache}\n'
+                      f'  gpg_pub_key = {gpg_pub_key}\n'
+                      f'===================================================')
+
+        # determine bitbake_args
+        self.bitbake_args = []
+        if not quiet:
+            self.bitbake_args.append('-v')
+
+        # write ci_build.conf
+        with open(self.build_dir + '/conf/ci_build.conf', 'w') as f:
+            if compat_arch:
+                f.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n')
+                f.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n')
+                f.write('ISAR_ENABLE_COMPAT_ARCH_debian-stretch_amd64 = "0"\n')
+                f.write('IMAGE_INSTALL += "kselftest"\n')
+            if cross:
+                f.write('ISAR_CROSS_COMPILE = "1"\n')
+            if debsrc_cache:
+                f.write('BASE_REPO_FEATURES = "cache-deb-src"\n')
+            if offline:
+                f.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
+                f.write('BB_NO_NETWORK = "1"\n')
+            if container:
+                f.write('SDK_FORMATS = "docker-archive"\n')
+                f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n')
+            if gpg_pub_key:
+                f.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
+            if distro_apt_premir:
+                f.write('DISTRO_APT_PREMIRRORS = "%s"\n' % distro_apt_premir)
+
+        # include ci_build.conf in local.conf
+        with open(self.build_dir + '/conf/local.conf', 'r+') as f:
+            for line in f:
+                if 'include ci_build.conf' in line:
+                    break
+            else:
+                f.write('\ninclude ci_build.conf')
+
+    def unconfigure(self):
+        open(self.build_dir + '/conf/ci_build.conf', 'w').close()
+
+    def delete_from_build_dir(self, path):
+        process.run('rm -rf ' + self.build_dir + '/' + path, sudo=True)
+
+    def bitbake(self, target, bitbake_cmd=None):
         self.log.info('===================================================')
-        self.log.info('Running ' + testname + ' test for:')
-        self.log.info(targets)
-        self.log.info('Isar build folder is: ' + build_dir)
+        self.log.info('Building ' + str(target))
         self.log.info('===================================================')
-
-        self.init(build_dir)
-        self.confprepare(build_dir, 1, cross, debsrc_cache)
-
-        return build_dir, bitbake_args;
-
-    def perform_build_test(self, targets, cross, bitbake_cmd):
-        build_dir, bb_args = self.prep('Isar build', targets, cross, 1)
-
-        self.log.info('Starting build...')
-
-        self.bitbake(build_dir, targets, bitbake_cmd, bb_args)
-
-    def perform_repro_test(self, targets, signed):
-        cross = int(self.params.get('cross', default=0))
-        build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0)
-
-        gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key'
-        gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key'
-
-        if signed:
-            with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-                # Enable use of signed cached base repository
-                file.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
-
-        os.chdir(build_dir)
-
-        os.environ['GNUPGHOME'] = tempfile.mkdtemp()
-        result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key))
-
-        if result.exit_status:
-            self.fail('GPG import failed')
-
-        self.bitbake(build_dir, targets, None, bb_args)
-
-        self.deletetmp(build_dir)
-        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-            file.write('ISAR_USE_CACHED_BASE_REPO = "1"\n')
-            file.write('BB_NO_NETWORK = "1"\n')
-
-        self.bitbake(build_dir, targets, None, bb_args)
-
-        # Disable use of cached base repository
-        self.confcleanup(build_dir)
-
-        if not signed:
-            # Try to build with changed configuration with no cleanup
-            self.bitbake(build_dir, targets, None, bb_args)
-
-        # Cleanup
-        self.deletetmp(build_dir)
-
-    def perform_wic_test(self, targets, wks_path, wic_path):
-        cross = int(self.params.get('cross', default=0))
-        build_dir, bb_args = self.prep('WIC exclude build', targets, cross, 1)
-
-        layerdir_isar = self.getlayerdir('isar')
-
-        wks_file = layerdir_isar + wks_path
-        wic_img = build_dir + wic_path
-
-        if not os.path.isfile(wic_img):
-            self.fail('No build started before: ' + wic_img + ' not exist')
-
-        self.backupfile(wks_file)
-        self.backupmove(wic_img)
-
-        with open(wks_file, 'r') as file:
-            lines = file.readlines()
-        with open(wks_file, 'w') as file:
-            for line in lines:
-                file.write(re.sub(r'part \/ ', 'part \/ --exclude-path usr ',
-                                  line))
-
+        os.chdir(self.build_dir)
+        cmdline = ['bitbake']
+        if self.bitbake_args:
+            cmdline.extend(self.bitbake_args)
+        if bitbake_cmd:
+            cmdline.append('-c')
+            cmdline.append(bitbake_cmd)
+        if isinstance(target, list):
+            cmdline.extend(target)
+        else:
+            cmdline.append(target)
+
+        with subprocess.Popen(" ".join(cmdline), stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE, universal_newlines=True,
+                              shell=True) as p1:
+            poller = select.poll()
+            poller.register(p1.stdout, select.POLLIN)
+            poller.register(p1.stderr, select.POLLIN)
+            while p1.poll() is None:
+                events = poller.poll(1000)
+                for fd, event in events:
+                    if event != select.POLLIN:
+                        continue
+                    if fd == p1.stdout.fileno():
+                        self.log.info(p1.stdout.readline().rstrip())
+                    if fd == p1.stderr.fileno():
+                        app_log.error(p1.stderr.readline().rstrip())
+            p1.wait()
+            if p1.returncode:
+                self.fail('Bitbake failed')
+
+    def backupfile(self, path):
         try:
-            self.bitbake(build_dir, targets, None, bb_args)
-        finally:
-            self.restorefile(wks_file)
+            shutil.copy2(path, path + backup_prefix)
+        except FileNotFoundError:
+            self.log.warn(path + ' not exist')
 
-        self.restorefile(wic_img)
-
-    def perform_container_test(self, targets, bitbake_cmd):
-        cross = int(self.params.get('cross', default=0))
-        build_dir, bb_args = self.prep('Isar Container', targets, cross, 1)
-
-        self.containerprep(build_dir)
-
-        self.bitbake(build_dir, targets, bitbake_cmd, bb_args)
-
-
-    def perform_ccache_test(self, targets):
-        build_dir, bb_args = self.prep('Isar ccache build', targets, 0, 0)
-
-        self.deletetmp(build_dir)
-        process.run('rm -rf ' + build_dir + '/ccache', sudo=True)
-
-        with open(build_dir + '/conf/ci_build.conf', 'a') as file:
-            file.write('USE_CCACHE = "1"\n')
-            file.write('CCACHE_TOP_DIR = "${TOPDIR}/ccache"')
+    def backupmove(self, path):
+        try:
+            shutil.move(path, path + backup_prefix)
+        except FileNotFoundError:
+            self.log.warn(path + ' not exist')
 
-        self.log.info('Starting build and filling ccache dir...')
-        start = time.time()
-        self.bitbake(build_dir, targets, None, bb_args)
-        first_time = time.time() - start
-        self.log.info('Non-cached build: ' + str(round(first_time)) + 's')
+    def restorefile(self, path):
+        try:
+            shutil.move(path + backup_prefix, path)
+        except FileNotFoundError:
+            self.log.warn(path + backup_prefix + ' not exist')
 
-        self.deletetmp(build_dir)
+    def getlayerdir(self, layer):
+        output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="')
+        env = dict(((x.split('=', 1) + [''])[:2] \
+                    for x in output.splitlines() if x != ''))
 
-        self.log.info('Starting build and using ccache dir...')
-        start = time.time()
-        self.bitbake(build_dir, targets, None, bb_args)
-        second_time = time.time() - start
-        self.log.info('Cached build: ' + str(round(second_time)) + 's')
+        return env['LAYERDIR_' + layer].strip('"')
 
-        speedup_k = 1.1
-        if first_time / second_time < speedup_k:
-            self.fail('No speedup after rebuild with ccache')
 
-        # Cleanup
-        self.deletetmp(build_dir)
-        process.run('rm -rf ' + build_dir + '/ccache', sudo=True)
diff --git a/testsuite/build_test/cibuilder.py b/testsuite/build_test/cibuilder.py
deleted file mode 100644
index 94786c7..0000000
--- a/testsuite/build_test/cibuilder.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-
-import logging
-import os
-import re
-import select
-import shutil
-import subprocess
-
-from avocado import Test
-from avocado.utils import path
-from avocado.utils import process
-
-isar_root = os.path.dirname(__file__) + '/../..'
-backup_prefix = '.ci-backup'
-
-app_log = logging.getLogger("avocado.app")
-
-class CIBuilder(Test):
-    def setUp(self):
-        super(CIBuilder, self).setUp()
-        job_log = os.path.join(os.path.dirname(self.logdir), '..', 'job.log')
-        self._file_handler = logging.FileHandler(filename=job_log)
-        self._file_handler.setLevel(logging.ERROR)
-        fmt = ('%(asctime)s %(module)-16.16s L%(lineno)-.4d %('
-               'levelname)-5.5s| %(message)s')
-        formatter = logging.Formatter(fmt=fmt)
-        self._file_handler.setFormatter(formatter)
-        app_log.addHandler(self._file_handler)
-
-    def init(self, build_dir):
-        os.chdir(isar_root)
-        path.usable_rw_dir(build_dir)
-        output = process.getoutput('/bin/bash -c "source isar-init-build-env \
-                                    %s 2>&1 >/dev/null; env"' % build_dir)
-        env = dict(((x.split('=', 1) + [''])[:2] \
-                    for x in output.splitlines() if x != ''))
-        os.environ.update(env)
-
-    def confprepare(self, build_dir, compat_arch, cross, debsrc_cache):
-        with open(build_dir + '/conf/ci_build.conf', 'w') as f:
-            if compat_arch:
-                f.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n')
-                f.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n')
-                f.write('ISAR_ENABLE_COMPAT_ARCH_debian-stretch_amd64 = "0"\n')
-                f.write('IMAGE_INSTALL += "kselftest"\n')
-            if cross:
-                f.write('ISAR_CROSS_COMPILE = "1"\n')
-            if debsrc_cache:
-                f.write('BASE_REPO_FEATURES = "cache-deb-src"\n')
-            distro_apt_premir = os.getenv('DISTRO_APT_PREMIRRORS')
-            if distro_apt_premir:
-                f.write('DISTRO_APT_PREMIRRORS = "%s"\n' % distro_apt_premir)
-
-        with open(build_dir + '/conf/local.conf', 'r+') as f:
-            for line in f:
-                if 'include ci_build.conf' in line:
-                    break
-            else:
-                f.write('\ninclude ci_build.conf')
-
-    def containerprep(self, build_dir):
-        with open(build_dir + '/conf/ci_build.conf', 'a') as f:
-            f.write('SDK_FORMATS = "docker-archive"\n')
-            f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n')
-
-    def confcleanup(self, build_dir):
-        open(build_dir + '/conf/ci_build.conf', 'w').close()
-
-    def deletetmp(self, build_dir):
-        process.run('rm -rf ' + build_dir + '/tmp', sudo=True)
-
-    def bitbake(self, build_dir, target, cmd, args):
-        os.chdir(build_dir)
-        cmdline = ['bitbake']
-        if args:
-            cmdline.append(args)
-        if cmd:
-            cmdline.append('-c')
-            cmdline.append(cmd)
-        if isinstance(target, list):
-            cmdline.extend(target)
-        else:
-            cmdline.append(target)
-
-        with subprocess.Popen(" ".join(cmdline), stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE, universal_newlines=True,
-                              shell=True) as p1:
-            poller = select.poll()
-            poller.register(p1.stdout, select.POLLIN)
-            poller.register(p1.stderr, select.POLLIN)
-            while p1.poll() is None:
-                events = poller.poll(1000)
-                for fd, event in events:
-                    if event != select.POLLIN:
-                        continue
-                    if fd == p1.stdout.fileno():
-                        self.log.info(p1.stdout.readline().rstrip())
-                    if fd == p1.stderr.fileno():
-                        app_log.error(p1.stderr.readline().rstrip())
-            p1.wait()
-            if p1.returncode:
-                self.fail('Bitbake failed')
-
-    def backupfile(self, path):
-        try:
-            shutil.copy2(path, path + backup_prefix)
-        except FileNotFoundError:
-            self.log.warn(path + ' not exist')
-
-    def backupmove(self, path):
-        try:
-            shutil.move(path, path + backup_prefix)
-        except FileNotFoundError:
-            self.log.warn(path + ' not exist')
-
-    def restorefile(self, path):
-        try:
-            shutil.move(path + backup_prefix, path)
-        except FileNotFoundError:
-            self.log.warn(path + backup_prefix + ' not exist')
-
-    def getlayerdir(self, layer):
-        try:
-            path.find_command('bitbake')
-        except path.CmdNotFoundError:
-            build_dir = self.params.get('build_dir',
-                                        default=isar_root + '/build')
-            self.init(build_dir)
-        output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="')
-        env = dict(((x.split('=', 1) + [''])[:2] \
-                    for x in output.splitlines() if x != ''))
-
-        return env['LAYERDIR_' + layer].strip('"')
-- 
2.30.2