[PATCH 0/2] Improve Isar CI logging

[PATCH 0/2] Improve Isar CI logging

[Anton Mikanovich]

Few changes inside Isar and testsuite to have better logged CI.

Anton Mikanovich (2):
  sbuild: Schroot config logging
  CI: Avoid removing logs from tmp

 meta/classes/sbuild.bbclass |  3 +++
 testsuite/cibase.py         | 16 ++++++++--------
 testsuite/citest.py         |  4 ++--
 3 files changed, 13 insertions(+), 10 deletions(-)

-- 
2.34.1

[PATCH 1/2] sbuild: Schroot config logging

[Anton Mikanovich]

To have additional debugging information print out any schroot config
files creation and removing logs.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 meta/classes/sbuild.bbclass | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/classes/sbuild.bbclass b/meta/classes/sbuild.bbclass
index 995f7a54..f1193c20 100644
--- a/meta/classes/sbuild.bbclass
+++ b/meta/classes/sbuild.bbclass
@@ -16,6 +16,7 @@ SBUILD_CONFIG="${WORKDIR}/sbuild.conf"
 
 schroot_create_configs() {
     mkdir -p "${TMPDIR}/schroot-overlay"
+    echo "Creating ${SCHROOT_CONF_FILE}"
     sudo -s <<'EOSUDO'
         set -e
 
@@ -56,8 +57,10 @@ schroot_delete_configs() {
     sudo -s <<'EOSUDO'
         set -e
         if [ -d "${SBUILD_CONF_DIR}" ]; then
+            echo "Removing ${SBUILD_CONF_DIR}"
             rm -rf "${SBUILD_CONF_DIR}"
         fi
+        echo "Removing ${SCHROOT_CONF_FILE}"
         rm -f "${SCHROOT_CONF_FILE}"
 EOSUDO
 }
-- 
2.34.1

[PATCH 2/2] CI: Avoid removing logs from tmp

[Anton Mikanovich]

All the tasks executed during CI run should left their logs. That's why
we can't just rm 'tmp' folders inside test cases. Move them in other
locations instead.
This will probably increase maximum disk usage but improve debugging.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 testsuite/cibase.py | 16 ++++++++--------
 testsuite/citest.py |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index adeff205..90591f32 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -45,7 +45,7 @@ class CIBaseTest(CIBuilder):
         try:
             self.bitbake(targets, **kwargs)
 
-            self.delete_from_build_dir('tmp')
+            self.move_in_build_dir('tmp', 'tmp_middle_repro_%s' % ('signed' if signed else 'unsigned'))
             self.configure(gpg_pub_key=gpg_pub_key if signed else None, offline=True, sstate_dir="", **kwargs)
 
             self.bitbake(targets, **kwargs)
@@ -80,7 +80,7 @@ class CIBaseTest(CIBuilder):
         # Field that stores direct ccache hits
         direct_cache_hit = 22
 
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_before_ccache')
         self.delete_from_build_dir('sstate-cache')
         self.delete_from_build_dir('ccache')
 
@@ -89,7 +89,7 @@ class CIBaseTest(CIBuilder):
         hit1 = ccache_stats(self.build_dir + '/ccache', direct_cache_hit)
         self.log.info('Ccache hits 1: ' + str(hit1))
 
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_middle_ccache')
         self.delete_from_build_dir('sstate-cache')
 
         self.log.info('Starting build and using ccache dir...')
@@ -101,7 +101,7 @@ class CIBaseTest(CIBuilder):
             self.fail('Ccache was not used on second build')
 
         # Cleanup
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_after_ccache')
         self.delete_from_build_dir('sstate-cache')
         self.delete_from_build_dir('ccache')
         self.unconfigure()
@@ -117,7 +117,7 @@ class CIBaseTest(CIBuilder):
 
         # Cleanup sstate and tmp before test
         self.delete_from_build_dir('sstate-cache')
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_before_sstate_populate')
 
         # Populate cache
         self.bitbake(image_target, **kwargs)
@@ -158,7 +158,7 @@ class CIBaseTest(CIBuilder):
         expected_files = set(glob.glob(f'{self.build_dir}/tmp/deploy/images/*/*'))
 
         # Rebuild image
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_before_sstate')
         self.bitbake(image_target, **kwargs)
         if not all([
                 check_executed_tasks('isar-bootstrap-target',
@@ -182,7 +182,7 @@ class CIBaseTest(CIBuilder):
             self.fail("Failed rebuild image")
 
         # Rebuild single package
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_middle_sstate')
         self.bitbake(package_target, **kwargs)
         if not all([
                 check_executed_tasks('isar-bootstrap-target',
@@ -195,7 +195,7 @@ class CIBaseTest(CIBuilder):
             self.fail("Failed rebuild single package")
 
         # Rebuild package and image
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_middle2_sstate')
         process.run(f'find {self.build_dir}/sstate-cache/ -name sstate:hello:* -delete')
         self.bitbake(image_target, **kwargs)
         if not all([
diff --git a/testsuite/citest.py b/testsuite/citest.py
index 7be2b86b..d7ece76c 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -162,7 +162,7 @@ class WicTest(CIBaseTest):
         targets = ['mc:qemuarm64-bookworm:isar-image-ci']
 
         self.init()
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_before_wic')
         self.perform_wic_partition_test(targets,
             wic_deploy_parts=False, debsrc_cache=True, compat_arch=False)
 
@@ -210,7 +210,7 @@ class NoCrossTest(CIBaseTest):
 
         self.init()
         # Cleanup after cross build
-        self.delete_from_build_dir('tmp')
+        self.move_in_build_dir('tmp', 'tmp_before_nocross')
         self.perform_build_test(targets, cross=False, debsrc_cache=True)
 
     def test_nocross_rpi(self):
-- 
2.34.1

Re: [PATCH 0/2] Improve Isar CI logging

[Uladzimir Bely]

On Tue, 2024-01-16 at 13:52 +0200, Anton Mikanovich wrote:
> Few changes inside Isar and testsuite to have better logged CI.
> 
> Anton Mikanovich (2):
>   sbuild: Schroot config logging
>   CI: Avoid removing logs from tmp
> 
>  meta/classes/sbuild.bbclass |  3 +++
>  testsuite/cibase.py         | 16 ++++++++--------
>  testsuite/citest.py         |  4 ++--
>  3 files changed, 13 insertions(+), 10 deletions(-)
> 
> -- 
> 2.34.1
> 
Applied to next.