From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6990633379674718208 X-Received: by 2002:a05:600c:3581:: with SMTP id p1mr1795779wmq.150.1627633677836; Fri, 30 Jul 2021 01:27:57 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:adf:db90:: with SMTP id u16ls1200817wri.2.gmail; Fri, 30 Jul 2021 01:27:56 -0700 (PDT) X-Google-Smtp-Source: ABdhPJw8oMpttMyMb2x2ib6Hb4T+EtGJpjXm6kIiDkqqPm5CnW0fS79UlOyFny4oDML0Yn5SO6hW X-Received: by 2002:adf:df12:: with SMTP id y18mr1710064wrl.189.1627633676799; Fri, 30 Jul 2021 01:27:56 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1627633676; cv=none; d=google.com; s=arc-20160816; b=OLfoGqY2sttEMoR6cFI5SefDiFp0S2fhzjRE7RUoCA+VXYfhM5GdtV0bBOdsQuOKnm H1WlixGxc+GWan0tOxC6i7raz9FkWEm5NBotLzO4NY/HKSedCaENAGOajy6Yqo1R4Gx3 W/o8mtzmSkkPkuZS433ICwUdI3gvO3HXevNSFonjJ8y2SZoi08cokM+PR8xFfitKBpH2 LIZr21qVuMYw9wsqPKvPn3E1Mpe8FcuI01wXmhM7pdVSk+unyPL/uBX6TDXFB9sPsNMk G+CkAtUdjXRH1Zqsx2r+LKi8IJtZOz5VlF7Wyx7lZPJHmWOj9uJdNBl3Nt3IEJgxcfKg 1x7Q== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=ktafKj+YAC/pAbQC7+irujfShOngtkQAhgNrYLE4RX4=; b=ewXq1zJ/CnTZMTShIgLLBMWZu0OOVfKsKZzv5TXi8lxhkAJksCum4i5wHGBM/xHBA8 jvRikIHFwHC5Hp+bEvmjJX6l8SMJqneUgBge9BxOEfrCuCcxtowZfWKWhSCvfDeyhasD l77zDTny5U7LdHiw2jfL0vwRkkbg2ws2bB7ftmB2Vsd4dbMCwXmQGhsv1LFjk9evXfCM LGUsRAr0aF45VkBgktWzVa58dGWc68tn8VwdVGE4aNnCarh+qj2fo9oBw3Vp3nXmrv90 2TpHzEkbx0di7qwgpD2lxMmoOnem4CWNikK4EZtiB8n0axI+KbzuCAgcI43WGekd10LD WTeA== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id c26si41180wml.0.2021.07.30.01.27.56 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Fri, 30 Jul 2021 01:27:56 -0700 (PDT) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (mm-88-30-214-37.mgts.dynamic.pppoe.byfly.by [37.214.30.88] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 16U8RkvQ008721 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Fri, 30 Jul 2021 10:27:55 +0200 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH v10 08/17] build_test: Refactoring build tests cases Date: Fri, 30 Jul 2021 11:27:30 +0300 Message-Id: <20210730082739.15042-9-amikan@ilbers.de> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210730082739.15042-1-amikan@ilbers.de> References: <20210730082739.15042-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: hB6VxYIewISm Split build test into classes and separate cases. Implement test cases tagging for external configuring. Also remove the usage of external varianter plugin. In this step, all ci_build test cases were copied as-is without functional changes. Signed-off-by: Anton Mikanovich --- testsuite/build_test/build_test.py | 208 ++++++++++++++++++++++++++--- testsuite/build_test/cibase.py | 110 +++++++++++++++ testsuite/build_test/cibuilder.py | 130 ++++++++++++++++++ testsuite/build_test/run.sh | 3 - testsuite/build_test/run_fast.sh | 3 + testsuite/build_test/run_full.sh | 3 + testsuite/build_test/variant.yaml | 22 --- 7 files changed, 434 insertions(+), 45 deletions(-) create mode 100644 testsuite/build_test/cibase.py create mode 100644 testsuite/build_test/cibuilder.py delete mode 100755 testsuite/build_test/run.sh create mode 100755 testsuite/build_test/run_fast.sh create mode 100755 testsuite/build_test/run_full.sh delete mode 100644 testsuite/build_test/variant.yaml diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py index 7a55c2f..9038ec5 100644 --- a/testsuite/build_test/build_test.py +++ b/testsuite/build_test/build_test.py @@ -1,29 +1,197 @@ #!/usr/bin/env python3 import os -import subprocess32 -import sys -from os.path import dirname -from avocado import Test +from avocado import skipUnless +from avocado.utils import path +from cibase import CIBaseTest -class BuildTest(Test): +UMOCI_AVAILABLE = True +SKOPEO_AVAILABLE = True +try: + path.find_command('umoci') +except path.CmdNotFoundError: + UMOCI_AVAILABLE = False +try: + path.find_command('skopeo') +except path.CmdNotFoundError: + SKOPEO_AVAILABLE = False - def test(self): - # TODO: add default values - build_dir = self.params.get('build_dir', default='.') - arch = self.params.get('arch', default='arm') - distro = self.params.get('distro', default='stretch') +class ReproTest(CIBaseTest): - self.log.info('===================================================') - self.log.info('Running Isar build test for (' + distro + '-' + arch + ')') - self.log.info('Isar build folder is: ' + build_dir) - self.log.info('===================================================') + """ + Test cached base repository - #isar_root = dirname(__file__) + '/..' - os.chdir(build_dir) - cmdline = ['bitbake', 'mc:qemu' + arch + '-' + distro + ':isar-image-base'] - p1 = subprocess32.run(cmdline) + :avocado: tags=repro,fast,full + """ + def test_repro_signed(self): + targets = [ + 'mc:de0-nano-soc-buster:isar-image-base', + 'mc:qemuarm64-stretch:isar-image-base' + ] - if p1.returncode: - self.fail('Test failed') + self.perform_repro_test(targets, 1) + + def test_repro_unsigned(self): + targets = [ + 'mc:qemuamd64-stretch:isar-image-base', + 'mc:qemuarm-stretch:isar-image-base' + ] + + self.perform_repro_test(targets, 0) + +class CrossTest(CIBaseTest): + + """ + Start cross build for the defined set of configurations + + :avocado: tags=cross,fast,full + """ + def test_cross(self): + targets = [ + 'mc:qemuarm-stretch:isar-image-base', + 'mc:qemuarm-buster:isar-image-base', + 'mc:qemuarm64-stretch:isar-image-base', + 'mc:qemuamd64-stretch:isar-image-base', + 'mc:de0-nano-soc-buster:isar-image-base', + 'mc:stm32mp15x-buster:isar-image-base', + 'mc:rpi-stretch:isar-image-base', + 'mc:qemuarm64-focal:isar-image-base' + ] + + self.perform_build_test(targets, 1, None) + + def test_cross_bullseye(self): + targets = [ + 'mc:qemuarm-bullseye:isar-image-base' + ] + + try: + self.perform_build_test(targets, 1, None) + except: + self.cancel('KFAIL') + +class SdkTest(CIBaseTest): + + """ + In addition test SDK creation + + :avocado: tags=sdk,fast,full + """ + def test_sdk(self): + targets = ['mc:qemuarm-stretch:isar-image-base'] + + self.perform_build_test(targets, 1, 'do_populate_sdk') + +class NoCrossTest(CIBaseTest): + + """ + Start non-cross build for the defined set of configurations + + :avocado: tags=nocross,full + """ + def test_nocross(self): + targets = [ + 'mc:qemuarm-stretch:isar-image-base', + 'mc:qemuarm-buster:isar-image-base', + 'mc:qemuarm64-stretch:isar-image-base', + 'mc:qemui386-stretch:isar-image-base', + 'mc:qemui386-buster:isar-image-base', + 'mc:qemuamd64-stretch:isar-image-base', + 'mc:qemuamd64-buster:isar-image-base', + 'mc:qemuamd64-buster-tgz:isar-image-base', + 'mc:qemuamd64-buster:isar-initramfs', + 'mc:qemumipsel-stretch:isar-image-base', + 'mc:qemumipsel-buster:isar-image-base', + 'mc:nand-ubi-demo-buster:isar-image-ubi', + 'mc:rpi-stretch:isar-image-base', + 'mc:qemuamd64-focal:isar-image-base' + ] + + # Cleanup after cross build + self.deletetmp(self.params.get('build_dir', + default=os.path.dirname(__file__) + '/../../build')) + + self.perform_build_test(targets, 0, None) + + def test_nocross_bullseye(self): + targets = [ + 'mc:qemuamd64-bullseye:isar-image-base', + 'mc:qemuarm-bullseye:isar-image-base', + 'mc:qemui386-bullseye:isar-image-base', + 'mc:qemumipsel-bullseye:isar-image-base' + ] + + try: + self.perform_build_test(targets, 0, None) + except: + self.cancel('KFAIL') + +class RebuildTest(CIBaseTest): + + """ + Test image rebuild + + :avocado: tags=rebuild,fast,full + """ + def test_rebuild(self): + is_cross_build = int(self.params.get('cross', default=0)) + + layerdir_core = self.getlayerdir('core') + + dpkgbase_file = layerdir_core + '/classes/dpkg-base.bbclass' + + self.backupfile(dpkgbase_file) + with open(dpkgbase_file, 'a') as file: + file.write('do_fetch_append() {\n\n}') + + try: + self.perform_build_test('mc:qemuamd64-stretch:isar-image-base', + is_cross_build, None) + finally: + self.restorefile(dpkgbase_file) + +class WicTest(CIBaseTest): + + """ + Test wic --exclude-path + + :avocado: tags=wic,fast,full + """ + def test_wic_exclude(self): + # TODO: remove hardcoded filenames + wks_path = '/scripts/lib/wic/canned-wks/sdimage-efi.wks' + wic_path = '/tmp/deploy/images/qemuamd64/isar-image-base-debian-stretch-qemuamd64.wic.img' + + self.perform_wic_test('mc:qemuamd64-stretch:isar-image-base', + wks_path, wic_path) + +class ContainerImageTest(CIBaseTest): + + """ + Test containerized images creation + + :avocado: tags=containerbuild,fast,full,container + """ + @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found') + def test_nocross(self): + targets = [ + 'mc:container-amd64-stretch:isar-image-base', + 'mc:container-amd64-buster:isar-image-base', + 'mc:container-amd64-bullseye:isar-image-base' + ] + + self.perform_container_test(targets, None) + +class ContainerSdkTest(CIBaseTest): + + """ + Test SDK container image creation + + :avocado: tags=containersdk,fast,full,container + """ + @skipUnless(UMOCI_AVAILABLE and SKOPEO_AVAILABLE, 'umoci/skopeo not found') + def test_container_sdk(self): + targets = ['mc:container-amd64-stretch:isar-image-base'] + + self.perform_container_test(targets, 'do_populate_sdk') diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py new file mode 100644 index 0000000..1a89f43 --- /dev/null +++ b/testsuite/build_test/cibase.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import os +import re +import tempfile + +from cibuilder import CIBuilder +from avocado.utils import process + +isar_root = os.path.dirname(__file__) + '/../..' + +class CIBaseTest(CIBuilder): + + def prep(self, testname, targets, cross, debsrc_cache): + build_dir = self.params.get('build_dir', default=isar_root + '/build') + build_dir = os.path.realpath(build_dir) + quiet = int(self.params.get('quiet', default=0)) + bitbake_args = '-v' + + if quiet: + bitbake_args = '' + + self.log.info('===================================================') + self.log.info('Running ' + testname + ' test for:') + self.log.info(targets) + self.log.info('Isar build folder is: ' + build_dir) + self.log.info('===================================================') + + self.init(build_dir) + self.confprepare(build_dir, 1, cross, debsrc_cache) + + return build_dir, bitbake_args; + + def perform_build_test(self, targets, cross, bitbake_cmd): + build_dir, bb_args = self.prep('Isar build', targets, cross, 1) + + self.log.info('Starting build...') + + self.bitbake(build_dir, targets, bitbake_cmd, bb_args) + + def perform_repro_test(self, targets, signed): + cross = int(self.params.get('cross', default=0)) + build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0) + + gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key' + gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key' + + if signed: + with open(build_dir + '/conf/ci_build.conf', 'a') as file: + # Enable use of signed cached base repository + file.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n') + + os.chdir(build_dir) + + os.environ['GNUPGHOME'] = tempfile.mkdtemp() + result = process.run('gpg --import %s %s' % (gpg_pub_key, gpg_priv_key)) + + if result.exit_status: + self.fail('GPG import failed') + + self.bitbake(build_dir, targets, None, bb_args) + + self.deletetmp(build_dir) + with open(build_dir + '/conf/ci_build.conf', 'a') as file: + file.write('ISAR_USE_CACHED_BASE_REPO = "1"\n') + file.write('BB_NO_NETWORK = "1"\n') + + self.bitbake(build_dir, targets, None, bb_args) + + # Cleanup and disable use of signed cached base repository + self.deletetmp(build_dir) + self.confcleanup(build_dir) + + def perform_wic_test(self, targets, wks_path, wic_path): + cross = int(self.params.get('cross', default=0)) + build_dir, bb_args = self.prep('WIC exclude build', targets, cross, 1) + + layerdir_isar = self.getlayerdir('isar') + + wks_file = layerdir_isar + wks_path + wic_img = build_dir + wic_path + + if not os.path.isfile(wic_img): + self.fail('No build started before: ' + wic_img + ' not exist') + + self.backupfile(wks_file) + self.backupmove(wic_img) + + with open(wks_file, 'r') as file: + lines = file.readlines() + with open(wks_file, 'w') as file: + for line in lines: + file.write(re.sub(r'part \/ ', 'part \/ --exclude-path usr ', + line)) + + try: + self.bitbake(build_dir, targets, None, bb_args) + finally: + self.restorefile(wks_file) + + self.restorefile(wic_img) + + def perform_container_test(self, targets, bitbake_cmd): + cross = int(self.params.get('cross', default=0)) + build_dir, bb_args = self.prep('Isar Container', targets, cross, 1) + + self.containerprep(build_dir) + + self.bitbake(build_dir, targets, bitbake_cmd, bb_args) + diff --git a/testsuite/build_test/cibuilder.py b/testsuite/build_test/cibuilder.py new file mode 100644 index 0000000..187363f --- /dev/null +++ b/testsuite/build_test/cibuilder.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +import logging +import os +import re +import select +import shutil +import subprocess + +from avocado import Test +from avocado.utils import path +from avocado.utils import process + +isar_root = os.path.dirname(__file__) + '/../..' +backup_prefix = '.ci-backup' + +app_log = logging.getLogger("avocado.app") + +class CIBuilder(Test): + def setUp(self): + super(CIBuilder, self).setUp() + job_log = os.path.join(os.path.dirname(self.logdir), '..', 'job.log') + self._file_handler = logging.FileHandler(filename=job_log) + self._file_handler.setLevel(logging.ERROR) + fmt = ('%(asctime)s %(module)-16.16s L%(lineno)-.4d %(' + 'levelname)-5.5s| %(message)s') + formatter = logging.Formatter(fmt=fmt) + self._file_handler.setFormatter(formatter) + app_log.addHandler(self._file_handler) + + def init(self, build_dir): + os.chdir(isar_root) + path.usable_rw_dir(build_dir) + output = process.getoutput('/bin/bash -c "source isar-init-build-env \ + %s 2>&1 >/dev/null; env"' % build_dir) + env = dict(((x.split('=', 1) + [''])[:2] \ + for x in output.splitlines() if x != '')) + os.environ.update(env) + + def confprepare(self, build_dir, compat_arch, cross, debsrc_cache): + with open(build_dir + '/conf/ci_build.conf', 'w') as f: + if compat_arch: + f.write('ISAR_ENABLE_COMPAT_ARCH_amd64 = "1"\n') + f.write('ISAR_ENABLE_COMPAT_ARCH_arm64 = "1"\n') + f.write('ISAR_ENABLE_COMPAT_ARCH_debian-stretch_amd64 = "0"\n') + f.write('IMAGE_INSTALL += "kselftest"\n') + if cross: + f.write('ISAR_CROSS_COMPILE = "1"\n') + if debsrc_cache: + f.write('BASE_REPO_FEATURES = "cache-deb-src"\n') + + with open(build_dir + '/conf/local.conf', 'r+') as f: + for line in f: + if 'include ci_build.conf' in line: + break + else: + f.write('\ninclude ci_build.conf') + + def containerprep(self, build_dir): + with open(build_dir + '/conf/ci_build.conf', 'a') as f: + f.write('SDK_FORMATS = "docker-archive"\n') + f.write('IMAGE_INSTALL_remove = "example-module-${KERNEL_NAME} enable-fsck"\n') + + def confcleanup(self, build_dir): + open(build_dir + '/conf/ci_build.conf', 'w').close() + + def deletetmp(self, build_dir): + process.run('rm -rf ' + build_dir + '/tmp', sudo=True) + + def bitbake(self, build_dir, target, cmd, args): + os.chdir(build_dir) + cmdline = ['bitbake'] + if args: + cmdline.append(args) + if cmd: + cmdline.append('-c') + cmdline.append(cmd) + if isinstance(target, list): + cmdline.extend(target) + else: + cmdline.append(target) + + with subprocess.Popen(" ".join(cmdline), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True, + shell=True) as p1: + poller = select.poll() + poller.register(p1.stdout, select.POLLIN) + poller.register(p1.stderr, select.POLLIN) + while p1.poll() is None: + events = poller.poll(1000) + for fd, event in events: + if fd == p1.stdout.fileno(): + self.log.info(p1.stdout.readline().rstrip()) + if fd == p1.stderr.fileno(): + app_log.error(p1.stderr.readline().rstrip()) + p1.wait() + if p1.returncode: + self.fail('Bitbake failed') + + def backupfile(self, path): + try: + shutil.copy2(path, path + backup_prefix) + except FileNotFoundError: + self.log.warn(path + ' not exist') + + def backupmove(self, path): + try: + shutil.move(path, path + backup_prefix) + except FileNotFoundError: + self.log.warn(path + ' not exist') + + def restorefile(self, path): + try: + shutil.move(path + backup_prefix, path) + except FileNotFoundError: + self.log.warn(path + backup_prefix + ' not exist') + + def getlayerdir(self, layer): + try: + path.find_command('bitbake') + except path.CmdNotFoundError: + build_dir = self.params.get('build_dir', + default=isar_root + '/build') + self.init(build_dir) + output = process.getoutput('bitbake -e | grep "^LAYERDIR_.*="') + env = dict(((x.split('=', 1) + [''])[:2] \ + for x in output.splitlines() if x != '')) + + return env['LAYERDIR_' + layer].strip('"') + diff --git a/testsuite/build_test/run.sh b/testsuite/build_test/run.sh deleted file mode 100755 index a8ea9cc..0000000 --- a/testsuite/build_test/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -avocado run build_test.py --mux-yaml test:variant.yaml --mux-inject build_dir:$BUILDDIR diff --git a/testsuite/build_test/run_fast.sh b/testsuite/build_test/run_fast.sh new file mode 100755 index 0000000..4d1bf44 --- /dev/null +++ b/testsuite/build_test/run_fast.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +avocado run build_test.py -t fast -p quiet=1 -p cross=1 diff --git a/testsuite/build_test/run_full.sh b/testsuite/build_test/run_full.sh new file mode 100755 index 0000000..af5ec59 --- /dev/null +++ b/testsuite/build_test/run_full.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +avocado run build_test.py -t full -p quiet=1 diff --git a/testsuite/build_test/variant.yaml b/testsuite/build_test/variant.yaml deleted file mode 100644 index 9ddc634..0000000 --- a/testsuite/build_test/variant.yaml +++ /dev/null @@ -1,22 +0,0 @@ -variants: !mux - stretch-amd64: - distro: "stretch" - arch: "amd64" - stretch-i386: - distro: "stretch" - arch: "i386" - stretch-arm: - distro: "stretch" - arch: "arm" - stretch-arm64: - distro: "stretch" - arch: "arm64" - buster-amd64: - distro: "buster" - arch: "amd64" - buster-i386: - distro: "buster" - arch: "i386" - buster-arm: - distro: "buster" - arch: "arm" -- 2.25.1