[PATCH 10/13] isar-sstate: add support to clean hashes from hashequiv server

["'Felix Moessbauer' via isar-users" <isar-users@googlegroups.com>]

The sstate cache and the hash equivalence data should be kept in sync
(in case hash-equivalence is used). For that, we a add a runtime option
to also clean the hashes from a running (external) hash equivalence
server.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 scripts/isar-sstate | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/scripts/isar-sstate b/scripts/isar-sstate
index 36a91134..d94091c8 100755
--- a/scripts/isar-sstate
+++ b/scripts/isar-sstate
@@ -49,6 +49,13 @@ files can be kept longer, as indicated by `--max-sig-age`. If not set explicitly
 this defaults to `max_age`, and any explicitly given value can't be smaller
 than `max_age`.
 
+If `--hashequiv-server` is given, the unihashes of the deleted archive files are
+also dropped from that hash equivalence server using `bitbake-hashclient remove`.
+Note, that the command has to be called from a bitbake environment. For a local
+sstate cache, start a local hash server and then run the clean command as follows:
+  bitbake-hashserv -d $BB_HASHSERVE_DB_DIR/hashserv.db &
+  isar-sstate clean <...> --hashequiv-server unix://.hashserve.sock $SSTATE_DIR
+
 ### info
 
 The `info` command scans the remote cache and displays some basic statistics.
@@ -145,6 +152,7 @@ from fnmatch import fnmatchcase
 import os
 import re
 import shutil
+import subprocess
 import sys
 from tempfile import NamedTemporaryFile
 import time
@@ -677,6 +685,10 @@ def arguments():
     parser.add_argument(
         '--excluded-tasks', type=str, default=DEFAULT_IGNORED_TASKS,
         help="lint: comma-separated list of tasks to ignore (default: %(default)s)")
+    parser.add_argument(
+        '--hashequiv-server', type=str, default=None,
+        help="clean: also remove the unihashes of deleted artifacts from this hashequiv server"
+    )
 
     args = parser.parse_args()
     if args.command in 'upload analyze'.split() and args.source is None:
@@ -734,7 +746,24 @@ def sstate_upload(source, target, verbose, filter, arch, **kwargs):
     return 0
 
 
-def sstate_clean(target, max_age, max_sig_age, verbose, filter, arch, **kwargs):
+def hashequiv_remove(server, unihashes, verbose):
+    """Drop the given unihashes from the hash equivalence server."""
+    hashclient = shutil.which('bitbake-hashclient')
+    if hashclient is None:
+        raise RuntimeError("bitbake-hashclient not found, cannot clean hashequiv server")
+
+    print(f"INFO: removing {len(unihashes)} unihashes from {server}")
+    for unihash in unihashes:
+        if verbose:
+            print(f"[UNIHASH] {unihash}")
+        cmd = [hashclient, '--address', server, 'remove', '--where', 'unihash', unihash]
+        try:
+            subprocess.run(cmd, capture_output=True, text=True, check=True)
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"could not remove unihash {unihash}: {e.stderr.strip()}") from e
+
+
+def sstate_clean(target, max_age, max_sig_age, verbose, filter, arch, hashequiv_server=None, **kwargs):
     def convert_to_seconds(x):
         seconds_per_unit = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400, 'w': 604800}
         m = re.match(r'^(\d+)(w|d|h|m|s)?', x)
@@ -776,6 +805,14 @@ def sstate_clean(target, max_age, max_sig_age, verbose, filter, arch, **kwargs):
     print(f"INFO: found {len(siginfo_files)} siginfo files, {len(del_siginfo_files)} of which "
           f"correspond to old archive files or are older than {max_sig_age}")
 
+    if hashequiv_server:
+        # drop the hashes first, the server must never point to artifacts we deleted
+        try:
+            hashequiv_remove(hashequiv_server, sorted(set(del_archive_hashes)), verbose)
+        except RuntimeError as e:
+            print(f"ERROR: {e}")
+            return 1
+
     for f in del_archive_files + del_siginfo_files:
         if verbose:
             print(f"[DELETE] {f.path}")
-- 
2.55.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20261002144936.246628-11-felix.moessbauer%40siemens.com.