[PATCH 1/2] ci: Rework ccache test

[Uladzimir Bely <ubely@ilbers.de>]

Previous ccache test implementation was based on comparing difference
between non-cached and cached build time. It is not reliable enough
because results may depend on overall CPU/disk load.

New implementation analyzes ccache statistics provided by */*/stats
files in ccache directory. Field with index 22 is used to check hits.

To reduce test time, we compile only `hello-isar` (and `libhello` as
the dependency) amd64 target during the test.

Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
 testsuite/cibase.py    | 33 ++++++++++++++++++++++++---------
 testsuite/cibuilder.py |  3 +++
 testsuite/citest.py    |  2 +-
 3 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/testsuite/cibase.py b/testsuite/cibase.py
index 2ffb8191..722d7bd2 100755
--- a/testsuite/cibase.py
+++ b/testsuite/cibase.py
@@ -46,32 +46,47 @@ class CIBaseTest(CIBuilder):
             self.bitbake(targets, **kwargs)
 
     def perform_ccache_test(self, targets, **kwargs):
+        def ccache_stats(dir, field):
+            # Look ccache source's 'src/core/Statistic.hpp' for field meanings
+            count = 0
+            for filename in glob.iglob(dir + '/**/stats', recursive=True):
+                if os.path.isfile(filename):
+                    with open(filename,'r') as file:
+                        content = file.readlines()
+                        if (field < len(content)):
+                            count += int(content[field])
+            return count
+
         self.configure(ccache=True, **kwargs)
 
+        # Field that stores direct ccache hits
+        direct_cache_hit = 22
+
         self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('sstate-cache')
         self.delete_from_build_dir('ccache')
 
         self.log.info('Starting build and filling ccache dir...')
-        start = time.time()
         self.bitbake(targets, **kwargs)
-        first_time = time.time() - start
-        self.log.info('Non-cached build: ' + str(round(first_time)) + 's')
+        hit1 = ccache_stats(self.build_dir + '/ccache', direct_cache_hit)
+        self.log.info('Ccache hits 1: ' + str(hit1))
 
         self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('sstate-cache')
 
         self.log.info('Starting build and using ccache dir...')
-        start = time.time()
         self.bitbake(targets, **kwargs)
-        second_time = time.time() - start
-        self.log.info('Cached build: ' + str(round(second_time)) + 's')
+        hit2 = ccache_stats(self.build_dir + '/ccache', direct_cache_hit)
+        self.log.info('Ccache hits 2: ' + str(hit2))
 
-        speedup_k = 1.1
-        if first_time / second_time < speedup_k:
-            self.fail('No speedup after rebuild with ccache')
+        if hit2 <= hit1:
+            self.fail('Ccache was not used on second build')
 
         # Cleanup
         self.delete_from_build_dir('tmp')
+        self.delete_from_build_dir('sstate-cache')
         self.delete_from_build_dir('ccache')
+        self.unconfigure()
 
     def perform_sstate_test(self, image_target, package_target, **kwargs):
         def check_executed_tasks(target, expected):
diff --git a/testsuite/cibuilder.py b/testsuite/cibuilder.py
index bc48d47f..dfb0a376 100755
--- a/testsuite/cibuilder.py
+++ b/testsuite/cibuilder.py
@@ -107,6 +107,9 @@ class CIBuilder(Test):
                 f.write('BASE_REPO_KEY="file://' + gpg_pub_key + '"\n')
             if distro_apt_premir:
                 f.write('DISTRO_APT_PREMIRRORS = "%s"\n' % distro_apt_premir)
+            if ccache:
+                f.write('USE_CCACHE = "1"\n')
+                f.write('CCACHE_TOP_DIR = "${TOPDIR}/ccache"\n')
 
         # include ci_build.conf in local.conf
         with open(self.build_dir + '/conf/local.conf', 'r+') as f:
diff --git a/testsuite/citest.py b/testsuite/citest.py
index 16e38d07..9135475e 100755
--- a/testsuite/citest.py
+++ b/testsuite/citest.py
@@ -56,7 +56,7 @@ class CcacheTest(CIBaseTest):
     :avocado: tags=ccache
     """
     def test_ccache_rebuild(self):
-        targets = ['mc:de0-nano-soc-bullseye:isar-image-base']
+        targets = ['mc:qemuamd64-bullseye:hello-isar']
         self.init()
         self.perform_ccache_test(targets)
 
-- 
2.20.1