RE: [PATCH v4 18/21] isar-sstate: adopt sstate maintenance script

["Schmidt, Adriaan" <adriaan.schmidt@siemens.com>]

Anton Mikanovich, Freitag, 5. August 2022 15:11:
> Fix sstate maintenance helper script to work with updated Bitbake
> siginfo data format.
> 
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
>  scripts/isar-sstate | 17 +++++++----------
>  1 file changed, 7 insertions(+), 10 deletions(-)
> 
> diff --git a/scripts/isar-sstate b/scripts/isar-sstate
> index 3a1945e..2b0addf 100755
> --- a/scripts/isar-sstate
> +++ b/scripts/isar-sstate
> @@ -126,7 +126,7 @@ import shutil
>  import sys
>  from tempfile import NamedTemporaryFile
>  import time
> -import pickle
> +import json
> 
>  sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)),
> '..', 'bitbake', 'lib'))
>  from bb.siggen import compare_sigfiles
> @@ -808,20 +808,17 @@ def sstate_lint(target, verbose, sources_dir,
> build_dir, exit_code, **kwargs):
>      hits_other = 0
>      for sig in cache_sigs.values():
>          sig_file = target.download(sig.path)
> -        with open(sig_file, 'rb') as f:
> -            try:
> -                sigdata_raw = pickle.Unpickler(f)
> -                sigdata = sigdata_raw.load()
> -            except:
> -                # invalid file or format... never mind
> -                continue
> +        with bb.compress.zstd.open(sig_file, "rt", encoding="utf-8",
> num_threads=1) as f:
> +            sigdata = json.load(f, object_hook=bb.siggen.SetDecoder)
> +
> +            bb.siggen.handle_renames(sigdata)

The try/catch introduced in a4df1824 was lost during rebase. 
The following patch (on top of yours) brings it back.

diff --git a/scripts/isar-sstate b/scripts/isar-sstate
index 2b0addf1..b6b5a1c2 100755
--- a/scripts/isar-sstate
+++ b/scripts/isar-sstate
@@ -808,32 +808,35 @@ def sstate_lint(target, verbose, sources_dir, build_dir, exit_code, **kwargs):
     hits_other = 0
     for sig in cache_sigs.values():
         sig_file = target.download(sig.path)
-        with bb.compress.zstd.open(sig_file, "rt", encoding="utf-8", num_threads=1) as f:
-            sigdata = json.load(f, object_hook=bb.siggen.SetDecoder)
-
+        try:
+            with bb.compress.zstd.open(sig_file, "rt", encoding="utf-8", num_threads=1) as f:
+                sigdata = json.load(f, object_hook=bb.siggen.SetDecoder)
             bb.siggen.handle_renames(sigdata)
+        except:
+            # invalid file or format... never mind
+            continue
 
-            pn_issues = []
-            for name, val in sigdata['varvals'].items():
-                if not name[0].isupper():
-                    continue
-                if sigdata['basehash_ignore_vars'] and name in sigdata['basehash_ignore_vars'] or \
-                   sigdata['taskhash_ignore_tasks'] and name in sigdata['taskhash_ignore_tasks'] or \
-                   name in ADDITIONAL_IGNORED_VARNAMES:
-                    continue
-                if not val or not val[0] == '/':
-                    continue
-                task = sigdata['task']
-                if val.startswith(build_dir):
-                    pn_issues.append(f'\033[0;31m-> path in build-dir:   {name} = "{val}"\033[0m')
-                    hits_builddir += 1
-                elif val.startswith(sources_dir):
-                    pn_issues.append(f'\033[0;31m-> path in sources-dir: {name} = "{val}"\033[0m')
-                    hits_srcdir += 1
-                else:
-                    hits_other += 1
-                    if verbose:
-                        pn_issues.append(f'\033[0;34m-> other absolute path: {name} = "{val}"\033[0m')
+        pn_issues = []
+        for name, val in sigdata['varvals'].items():
+            if not name[0].isupper():
+                continue
+            if sigdata['basehash_ignore_vars'] and name in sigdata['basehash_ignore_vars'] or \
+               sigdata['taskhash_ignore_tasks'] and name in sigdata['taskhash_ignore_tasks'] or \
+               name in ADDITIONAL_IGNORED_VARNAMES:
+                continue
+            if not val or not val[0] == '/':
+                continue
+            task = sigdata['task']
+            if val.startswith(build_dir):
+                pn_issues.append(f'\033[0;31m-> path in build-dir:   {name} = "{val}"\033[0m')
+                hits_builddir += 1
+            elif val.startswith(sources_dir):
+                pn_issues.append(f'\033[0;31m-> path in sources-dir: {name} = "{val}"\033[0m')
+                hits_srcdir += 1
+            else:
+                hits_other += 1
+                if verbose:
+                    pn_issues.append(f'\033[0;34m-> other absolute path: {name} = "{val}"\033[0m')
             if len(pn_issues) > 0:
                 print(f"\033[1;33m==== issues found in {sig.arch}:{sig.pn}:{sig.task} ({sig.hash[:8]}) ====\033[0m")
                 print('\n'.join(pn_issues))

Adriaan

> 
>              pn_issues = []
>              for name, val in sigdata['varvals'].items():
>                  if not name[0].isupper():
>                      continue
> -                if sigdata['basewhitelist'] and name in
> sigdata['basewhitelist'] or \
> -                   sigdata['taskwhitelist'] and name in
> sigdata['taskwhitelist'] or \
> +                if sigdata['basehash_ignore_vars'] and name in
> sigdata['basehash_ignore_vars'] or \
> +                   sigdata['taskhash_ignore_tasks'] and name in
> sigdata['taskhash_ignore_tasks'] or \
>                     name in ADDITIONAL_IGNORED_VARNAMES:
>                      continue
>                  if not val or not val[0] == '/':
> --
> 2.17.1
> 
> --
> You received this message because you are subscribed to the Google Groups
> "isar-users" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to isar-users+unsubscribe@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/isar-users/20220805131035.22844-19-
> amikan%40ilbers.de.