[PATCH 2/3] isar: Add dependency explorer

[Anton Mikanovich <amikan@ilbers.de>]

Explore Isar recipes for any build dependencies mentioned in source
packages for not actually added at recipes.

Usage (inside build dir):
$ ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base

After that normal Isar build can be performed.

This script requires python3-apt to be installed.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 scripts/explore.py | 175 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100755 scripts/explore.py

diff --git a/scripts/explore.py b/scripts/explore.py
new file mode 100755
index 00000000..56c7378a
--- /dev/null
+++ b/scripts/explore.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import apt_pkg
+import apt.progress.base
+
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/../bitbake/lib')
+
+import bb.tinfoil
+
+class IsarBuilder:
+    def __init__(self):
+        self.tinfoil = bb.tinfoil.Tinfoil()
+        self.tinfoil.prepare()
+
+    def create_apt_rootfs(self, rootfs):
+        repo_isar_dir = '%s/tmp/deploy/isar-apt/%s-%s/apt' % (os.path.abspath(os.getcwd()), self.distro, self.arch)
+        mirror = 'file://%s/%s' % (repo_isar_dir, self.distro)
+        codename = 'isar'
+
+        if not os.path.exists(rootfs + '/var/lib/dpkg'):
+            os.makedirs(rootfs + '/var/lib/dpkg')
+        open(rootfs + '/var/lib/dpkg/status', 'w').close()
+
+        if not os.path.exists(rootfs + '/etc/apt/preferences.d'):
+            os.makedirs(rootfs + '/etc/apt/preferences.d')
+        with open(rootfs + '/etc/apt/sources.list', 'w') as f:
+            f.write('deb [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+            f.write('deb-src [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+
+        if not os.path.exists(rootfs + '/var/cache/apt/archives/partial'):
+            os.makedirs(rootfs + '/var/cache/apt/archives/partial')
+
+    def apt_config(self):
+        apt_pkg.init()
+
+        rootfs = os.path.abspath(os.getcwd()) + '/tmp/deps_poc_rootfs/%s-%s' % (self.distro, self.arch)
+
+        if not os.path.isdir(rootfs):
+            self.create_apt_rootfs(rootfs)
+
+        apt_pkg.config.set('APT::Architecture', self.arch)
+        apt_pkg.config.set('Dir', rootfs)
+        apt_pkg.config.set('Dir::Cache', rootfs + '/var/cache/apt')
+        apt_pkg.config.set('Dir::State::status', rootfs + '/var/lib/dpkg/status')
+
+        apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
+
+    def isar_apt_update(self):
+        sources = apt_pkg.SourceList()
+        sources.read_main_list()
+
+        progress = apt.progress.text.AcquireProgress()
+
+        self.cache = apt_pkg.Cache()
+        self.cache.update(progress, sources)
+        self.cache = apt_pkg.Cache()
+
+    def bitbake(self, targets, task=''):
+        targets = [targets] if isinstance(targets, str) else targets
+        target_str = ''
+        for pn in targets:
+            if self.mc:
+                pn = 'mc:%s:%s' % (self.mc, pn)
+            target_str += '%s ' % (pn if not task else ':'.join([pn, task]))
+        targets.clear()
+        if target_str != '':
+            print('Building %s' % target_str)
+            return self.tinfoil.build_targets(target_str)
+        else:
+            return True
+
+    def deps_lookup(self, pkg, parent_provider, add=True):
+        if pkg in self.cache:
+            return
+
+        package_target = pkg if not self.mc else ':'.join(['mc', self.mc, pkg])
+
+        provider = self.tinfoil.find_best_provider(package_target)
+        if provider[3] is None or provider[3].endswith('linux-distro.bb'):
+            return
+
+        local_d = self.tinfoil.parse_recipe(package_target)
+        source_task = local_d.getVar('do_deploy_source', expand=True)
+
+        declared_deps = self.recipecache.deps[provider[3]]
+        build_deps = []
+
+        if hasattr(self, 'sr'):
+            self.sr.restart()
+
+            if self.sr.lookup(pkg):
+                if 'Build-Depends' in self.sr.build_depends:
+                    for dep in self.sr.build_depends['Build-Depends']:
+                        child = str(dep[0][0])
+                        if child not in build_deps:
+                            build_deps.append(child)
+            elif parent_provider and source_task:
+                self.need_source.append(pkg)
+        elif source_task:
+            self.need_source.append(pkg)
+
+        if parent_provider and add:
+            print(parent_provider[3] + ' depends on ' + pkg)
+            append_filename = parent_provider[3].rsplit(':', 1)[-1] + 'append'
+            with open(append_filename, 'a') as f:
+                f.write(f'DEPENDS += "{pkg}"\n\r')
+
+        for dep in declared_deps:
+            self.deps_lookup(dep, provider, False)
+        for dep in list(set(build_deps) - set(declared_deps)):
+            self.deps_lookup(dep, provider)
+
+    def init(self, mc, target):
+        self.mc = mc
+        self.target = target
+
+        package_target = target if not self.mc else ':'.join(['mc', self.mc, target])
+
+        self.d = self.tinfoil.parse_recipe(package_target)
+        self.distro = self.d.getVar('DISTRO', expand=True)
+        self.arch = self.d.getVar('DISTRO_ARCH', expand=True)
+
+        self.recipecache = self.tinfoil.cooker.recipecaches[mc]
+        self.apt_config()
+
+        self.need_source = []
+
+    def shutdown(self):
+        self.tinfoil.shutdown()
+
+    def explore_deps(self):
+        maxdepth = 10
+        while maxdepth > 0:
+            maxdepth -= 1
+
+            try:
+                self.isar_apt_update()
+                self.sr = apt_pkg.SourceRecords()
+            except:
+                print('No apt-cache found')
+
+            self.deps_lookup(self.target, None)
+
+            if not self.need_source:
+                break
+
+            print('Need sources for ' + str(self.need_source))
+
+            if not self.bitbake(self.need_source, 'do_deploy_source'):
+                break
+
+        if self.need_source:
+            print('Following packages still left unchecked: ' + str(self.need_source))
+
+if len(sys.argv) != 3:
+    print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base')
+    exit(1)
+
+mc = ''
+if sys.argv[2].startswith('mc:'):
+    args = sys.argv[2].split(':')
+    mc = args[1]
+    target = args[2]
+else:
+    args = sys.argv[2].split(':')
+    target = sys.argv[2]
+
+ib = IsarBuilder()
+try:
+    ib.init(mc, target)
+    ib.explore_deps()
+finally:
+    ib.shutdown()
-- 
2.34.1