From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7016230395466219520 X-Received: by 2002:a2e:9889:: with SMTP id b9mr545405ljj.167.1637081214628; Tue, 16 Nov 2021 08:46:54 -0800 (PST) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:651c:201a:: with SMTP id s26ls2968023ljo.5.gmail; Tue, 16 Nov 2021 08:46:53 -0800 (PST) X-Google-Smtp-Source: ABdhPJyE2C49UYxRiKPSClpQaIrCEIAV6ir46tuLvFNKRIUpHF4LpYyjXQdzZGNduJbjqVARu8OK X-Received: by 2002:a2e:9654:: with SMTP id z20mr568264ljh.254.1637081213541; Tue, 16 Nov 2021 08:46:53 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1637081213; cv=none; d=google.com; s=arc-20160816; b=C7vRLhqq1Sea8tihfqtdb1OGJ8Ux4EL/+9F14qcPLH11L8DP475naYBDKWPpFxv2lZ NCNx6DHCoKYh14BbedrPoOkRAW80YvTLcgn0weUGzCcXlWQ1MEQOP6noeU4qMea9FOnl pJkxm5D5yrEfh8zh+1esFYBqeteewdgGSYZ5gV2r8ATFSZFA174uGzZvDSBupwzFH62r EeF3HUde/M2so1d0czUr37dcSTAh57PaDd9FmXihi/yf8zSfH4YaNSBML7Wp5zTwLWon q+FBS64b1yxiMPhoY0CxW3V+WK864WGd7wPb3Uaa+CblBDdQY6ExhckqLk63jfZD6rXa vlvw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-language:content-transfer-encoding:in-reply-to:mime-version :user-agent:date:message-id:from:references:to:subject; bh=OzubmqqInijEGTCs2DEO6U+dEuQEvoaE65r7/o7KPO4=; b=0nBYhm8pTapf4FP70VzYtH+FujzR0c0gIYiEYJmYLgR2/nmLW0tZ6u89VwK5B2y07O 9qeFFj3/9XVoz/SU0/IPys9pfEYeWR+a7Sn0nbcXvlMr4MMRxrlWYNsFfB3a6+Lq/8hx 9WVjfajvhkJ1Vq3jKuRk9/cn+GXWxriU4IWpX8RV8VchqQlxqnG9QGm4CpKEWZwWev5g JOxzIYfTJHvMuxgDAV67MIPx8NTOOz2W386irJT47VtiCykLFfUc1cnXRc67UQ+hL+Pd L9haTJNHWH1ktyez7lJW/ROltFFcRZltdI0HoEfe5iZk+y20+LVxKxIz6tMDARc2M8l9 kpHA== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id h12si120465lfv.4.2021.11.16.08.46.53 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Tue, 16 Nov 2021 08:46:53 -0800 (PST) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from [192.168.67.164] (mm-96-28-214-37.mgts.dynamic.pppoe.byfly.by [37.214.28.96] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 1AGGkpwJ007733 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Tue, 16 Nov 2021 17:46:52 +0100 Subject: Re: [PATCH v6 13/13] sstate: add test case To: Adriaan Schmidt , isar-users@googlegroups.com References: <20211111064728.2375760-1-adriaan.schmidt@siemens.com> <20211111064728.2375760-14-adriaan.schmidt@siemens.com> From: Anton Mikanovich Message-ID: Date: Tue, 16 Nov 2021 19:46:45 +0300 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.13.0 MIME-Version: 1.0 In-Reply-To: <20211111064728.2375760-14-adriaan.schmidt@siemens.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit Content-Language: en-US X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: KbGmFNiF8LUx 11.11.2021 09:47, Adriaan Schmidt wrote: > This also adds a parameter `sstate` to the build function, which is > only passed as `1` for the actual sstate test. None of the other tests > uses sstate caching. > > Signed-off-by: Adriaan Schmidt > --- > testsuite/build_test/build_test.py | 101 ++++++++++++++++++++++++++--- > testsuite/build_test/cibase.py | 14 ++-- > 2 files changed, 101 insertions(+), 14 deletions(-) > > diff --git a/testsuite/build_test/build_test.py b/testsuite/build_test/build_test.py > index d39c10c..817f4ce 100644 > --- a/testsuite/build_test/build_test.py > +++ b/testsuite/build_test/build_test.py > @@ -1,9 +1,10 @@ > #!/usr/bin/env python3 > > +import glob > import os > > from avocado import skipUnless > -from avocado.utils import path > +from avocado.utils import path, process > from cibase import CIBaseTest > > UMOCI_AVAILABLE = True > @@ -58,7 +59,7 @@ class CrossTest(CIBaseTest): > 'mc:rpi-stretch:isar-image-base' > ] > > - self.perform_build_test(targets, 1, None) > + self.perform_build_test(targets, 1, 0, None) > > def test_cross_ubuntu(self): > targets = [ > @@ -66,7 +67,7 @@ class CrossTest(CIBaseTest): > ] > > try: > - self.perform_build_test(targets, 1, None) > + self.perform_build_test(targets, 1, 0, None) > except: > self.cancel('KFAIL') > > @@ -76,7 +77,7 @@ class CrossTest(CIBaseTest): > ] > > try: > - self.perform_build_test(targets, 1, None) > + self.perform_build_test(targets, 1, 0, None) > except: > self.cancel('KFAIL') > > @@ -90,7 +91,7 @@ class SdkTest(CIBaseTest): > def test_sdk(self): > targets = ['mc:qemuarm-stretch:isar-image-base'] > > - self.perform_build_test(targets, 1, 'do_populate_sdk') > + self.perform_build_test(targets, 1, 0, 'do_populate_sdk') > > class NoCrossTest(CIBaseTest): > > @@ -123,7 +124,7 @@ class NoCrossTest(CIBaseTest): > self.deletetmp(self.params.get('build_dir', > default=os.path.dirname(__file__) + '/../../build')) > > - self.perform_build_test(targets, 0, None) > + self.perform_build_test(targets, 0, 0, None) > > def test_nocross_bullseye(self): > targets = [ > @@ -134,7 +135,7 @@ class NoCrossTest(CIBaseTest): > ] > > try: > - self.perform_build_test(targets, 0, None) > + self.perform_build_test(targets, 0, 0, None) > except: > self.cancel('KFAIL') > > @@ -158,7 +159,7 @@ class RebuildTest(CIBaseTest): > > try: > self.perform_build_test('mc:qemuamd64-stretch:isar-image-base', > - is_cross_build, None) > + is_cross_build, 0, None) > finally: > self.restorefile(dpkgbase_file) > > @@ -206,3 +207,87 @@ class ContainerSdkTest(CIBaseTest): > targets = ['mc:container-amd64-stretch:isar-image-base'] > > self.perform_container_test(targets, 'do_populate_sdk') > + > +class SstateTest(CIBaseTest): > + > + """ > + Test builds with artifacts taken from sstate cache > + > + :avocado: tags=sstate,fast,full > + """ > + def check_executed_tasks(self, build_dir, target, expected): > + taskorder_file = glob.glob(f'{build_dir}/tmp/work/{self.distro_distroarch}/{target}/*/temp/log.task_order') > + try: > + with open(taskorder_file[0], 'r') as f: > + tasks = [l.split()[0] for l in f.readlines()] > + except (FileNotFoundError, IndexError): > + tasks = [] > + if expected is None: > + # require that no tasks were executed > + return len(tasks) == 0 > + for e in expected: > + should_run = True > + if e.startswith('!'): > + should_run = False > + e = e[1:] > + if should_run != (e in tasks): > + self.log.error(f"{target}: executed tasks {str(tasks)} did not match expected {str(expected)}") > + return False > + return True > + > + def test_sstate(self): > + build_dir, bb_args = self.prep('Sstate', '', 0, 1, 1) > + image_target = 'mc:qemuamd64-bullseye:isar-image-base' > + package_target = 'mc:qemuamd64-bullseye:hello' > + self.distro_distroarch = 'debian-bullseye-amd64' > + > + # Cleanup sstate and tmp before test > + process.run(f'rm -rf {build_dir}/sstate-cache', sudo=True) > + self.deletetmp(build_dir) > + > + # Populate cache > + self.perform_build_test(image_target, 0, 1, None) > + > + # Rebuild image > + self.deletetmp(build_dir) > + self.perform_build_test(image_target, 0, 1, None) > + if not all([ > + self.check_executed_tasks(build_dir, 'isar-bootstrap-target', > + ['do_bootstrap_setscene', '!do_bootstrap']), > + self.check_executed_tasks(build_dir, 'buildchroot-target', > + ['do_rootfs_setscene', '!do_rootfs']), > + self.check_executed_tasks(build_dir, 'isar-image-base-*-wic-img', > + ['do_rootfs_setscene', '!do_rootfs']) > + ]): > + self.fail("Failed rebuild image") > + > + # Rebuild single package > + self.deletetmp(build_dir) > + self.perform_build_test(package_target, 0, 1, None) > + if not all([ > + self.check_executed_tasks(build_dir, 'isar-bootstrap-target', > + ['do_bootstrap_setscene']), > + self.check_executed_tasks(build_dir, 'buildchroot-target', > + ['!do_buildchroot_deploy']), > + self.check_executed_tasks(build_dir, 'hello', > + ['do_dpkg_build_setscene', 'do_deploy_deb', '!do_dpkg_build']) > + ]): > + self.fail("Failed rebuild single package") > + > + # Rebuild package and image > + self.deletetmp(build_dir) > + process.run(f'find {build_dir}/sstate-cache/ -name sstate:hello:* -delete') > + self.perform_build_test(image_target, 0, 1, None) > + if not all([ > + self.check_executed_tasks(build_dir, 'isar-bootstrap-target', > + ['do_bootstrap_setscene', '!do_bootstrap']), > + self.check_executed_tasks(build_dir, 'buildchroot-target', > + ['do_rootfs_setscene', '!do_rootfs']), > + self.check_executed_tasks(build_dir, 'hello', > + ['do_fetch', 'do_dpkg_build']), > + # TODO: if we actually make a change to hello, then we could test that do_rootfs is executed. > + # currently, hello is rebuilt, but its sstate sig/hash does not change. > + self.check_executed_tasks(build_dir, 'isar-image-base-*-wic-img', > + ['do_rootfs_setscene', '!do_rootfs']) > + ]): > + self.fail("Failed rebuild package and image") > diff --git a/testsuite/build_test/cibase.py b/testsuite/build_test/cibase.py > index 0bff7e4..3cba412 100644 > --- a/testsuite/build_test/cibase.py > +++ b/testsuite/build_test/cibase.py > @@ -11,7 +11,7 @@ isar_root = os.path.dirname(__file__) + '/../..' > > class CIBaseTest(CIBuilder): > > - def prep(self, testname, targets, cross, debsrc_cache): > + def prep(self, testname, targets, cross, debsrc_cache, sstate): > build_dir = self.params.get('build_dir', default=isar_root + '/build') > build_dir = os.path.realpath(build_dir) > quiet = int(self.params.get('quiet', default=0)) > @@ -19,6 +19,8 @@ class CIBaseTest(CIBuilder): > > if not quiet: > bitbake_args.append('-v') > + if not sstate: > + bitbake_args.append('--no-setscene') > > self.log.info('===================================================') > self.log.info('Running ' + testname + ' test for:') > @@ -31,8 +33,8 @@ class CIBaseTest(CIBuilder): > > return build_dir, bitbake_args; > > - def perform_build_test(self, targets, cross, bitbake_cmd): > - build_dir, bb_args = self.prep('Isar build', targets, cross, 1) > + def perform_build_test(self, targets, cross, sstate, bitbake_cmd): > + build_dir, bb_args = self.prep('Isar build', targets, cross, 1, sstate) > > self.log.info('Starting build...') > > @@ -40,7 +42,7 @@ class CIBaseTest(CIBuilder): > > def perform_repro_test(self, targets, signed): > cross = int(self.params.get('cross', default=0)) > - build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0) > + build_dir, bb_args = self.prep('repro Isar build', targets, cross, 0, 0) > > gpg_pub_key = os.path.dirname(__file__) + '/../base-apt/test_pub.key' > gpg_priv_key = os.path.dirname(__file__) + '/../base-apt/test_priv.key' > @@ -79,7 +81,7 @@ class CIBaseTest(CIBuilder): > > def perform_wic_test(self, targets, wks_path, wic_path): > cross = int(self.params.get('cross', default=0)) > - build_dir, bb_args = self.prep('WIC exclude build', targets, cross, 1) > + build_dir, bb_args = self.prep('WIC exclude build', targets, cross, 1, 0) > > layerdir_isar = self.getlayerdir('isar') > > @@ -108,7 +110,7 @@ class CIBaseTest(CIBuilder): > > def perform_container_test(self, targets, bitbake_cmd): > cross = int(self.params.get('cross', default=0)) > - build_dir, bb_args = self.prep('Isar Container', targets, cross, 1) > + build_dir, bb_args = self.prep('Isar Container', targets, cross, 1, 0) > > self.containerprep(build_dir) > Thanks for adding test case, but if you need to add it to fast/full in should be moved to the beginning of the build_test.py (after repro maybe) to not cleanup images needed for start_vm testing. Executing it at the end will also remove all the logs of common tests. Maybe we can not remove tmp but just set any empty SSTATE_DIR? Changing testsuite API should probably be moved to the separate patch. -- Anton Mikanovich Promwad Ltd. External service provider of ilbers GmbH Maria-Merian-Str. 8 85521 Ottobrunn, Germany +49 (89) 122 67 24-0 Commercial register Munich, HRB 214197 General Manager: Baurzhan Ismagulov