[PATCH 0/3] Introduce Debian dependencies and provides helper

[PATCH 0/3] Introduce Debian dependencies and provides helper

[Anton Mikanovich]

Package information from debian/control file is not available when
bitbake prepares dependency tree on beginning of the build operation.
So developers should manually set at least DEPENDS and PROVIDES values.

This helper script can do the following:
1. Scan for all recipes and add missed PROVIDES values.
2. Scan for target dependencies and add missed DEPENDS values.

Missed values are added as bbappend files near the original recipes.

Usage:
$ ../scripts/explore.py [ all | deps | provides ] mc:qemuamd64-bullseye:isar-image-base

It supposed to be executed inside build dir and requires python3-apt package to
be installed. Running PROVIDES scanning also requires COMPATIBLE_MACHINE to be
used in machine-specific recipes, so there will be no way to build them for
incompatible target.

This patchset should be placed on top of:
[PATCH 0/5] Separate prebuild activities

Anton Mikanovich (3):
  conf: Enable bbappends
  isar: Add dependency explorer
  isar: Add provides explorer

 meta-isar/conf/layer.conf |   2 +-
 meta-test/conf/layer.conf |   2 +-
 meta/conf/layer.conf      |   2 +-
 scripts/explore.py        | 264 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 267 insertions(+), 3 deletions(-)
 create mode 100755 scripts/explore.py

-- 
2.34.1

[PATCH 1/3] conf: Enable bbappends

[Anton Mikanovich]

Enable usage of bbappend files in all the layers.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 meta-isar/conf/layer.conf | 2 +-
 meta-test/conf/layer.conf | 2 +-
 meta/conf/layer.conf      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/meta-isar/conf/layer.conf b/meta-isar/conf/layer.conf
index dec2658f..2209b18b 100644
--- a/meta-isar/conf/layer.conf
+++ b/meta-isar/conf/layer.conf
@@ -5,7 +5,7 @@
 BBPATH .= ":${LAYERDIR}"
 
 # We have recipes-* directories, add to BBFILES
-BBFILES +=  "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
 
 BBFILE_COLLECTIONS += "isar"
 BBFILE_PATTERN_isar = "^${LAYERDIR}/"
diff --git a/meta-test/conf/layer.conf b/meta-test/conf/layer.conf
index 07b5c77d..0f15119d 100644
--- a/meta-test/conf/layer.conf
+++ b/meta-test/conf/layer.conf
@@ -5,7 +5,7 @@
 BBPATH .= ":${LAYERDIR}"
 
 # We have recipes-* directories, add to BBFILES
-BBFILES +=  "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
 
 BBFILE_COLLECTIONS += "test"
 BBFILE_PATTERN_test = "^${LAYERDIR}/"
diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index a38ab97d..b622ce69 100644
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -5,7 +5,7 @@
 BBPATH .= ":${LAYERDIR}"
 
 # We have recipes-* directories, add to BBFILES
-BBFILES +=  "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
 
 BBFILE_COLLECTIONS += "core"
 BBFILE_PATTERN_core = "^${LAYERDIR}/"
-- 
2.34.1

[PATCH 2/3] isar: Add dependency explorer

[Anton Mikanovich]

Explore Isar recipes for any build dependencies mentioned in source
packages for not actually added at recipes.

Usage (inside build dir):
$ ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base

After that normal Isar build can be performed.

This script requires python3-apt to be installed.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 scripts/explore.py | 175 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100755 scripts/explore.py

diff --git a/scripts/explore.py b/scripts/explore.py
new file mode 100755
index 00000000..56c7378a
--- /dev/null
+++ b/scripts/explore.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import apt_pkg
+import apt.progress.base
+
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/../bitbake/lib')
+
+import bb.tinfoil
+
+class IsarBuilder:
+    def __init__(self):
+        self.tinfoil = bb.tinfoil.Tinfoil()
+        self.tinfoil.prepare()
+
+    def create_apt_rootfs(self, rootfs):
+        repo_isar_dir = '%s/tmp/deploy/isar-apt/%s-%s/apt' % (os.path.abspath(os.getcwd()), self.distro, self.arch)
+        mirror = 'file://%s/%s' % (repo_isar_dir, self.distro)
+        codename = 'isar'
+
+        if not os.path.exists(rootfs + '/var/lib/dpkg'):
+            os.makedirs(rootfs + '/var/lib/dpkg')
+        open(rootfs + '/var/lib/dpkg/status', 'w').close()
+
+        if not os.path.exists(rootfs + '/etc/apt/preferences.d'):
+            os.makedirs(rootfs + '/etc/apt/preferences.d')
+        with open(rootfs + '/etc/apt/sources.list', 'w') as f:
+            f.write('deb [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+            f.write('deb-src [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+
+        if not os.path.exists(rootfs + '/var/cache/apt/archives/partial'):
+            os.makedirs(rootfs + '/var/cache/apt/archives/partial')
+
+    def apt_config(self):
+        apt_pkg.init()
+
+        rootfs = os.path.abspath(os.getcwd()) + '/tmp/deps_poc_rootfs/%s-%s' % (self.distro, self.arch)
+
+        if not os.path.isdir(rootfs):
+            self.create_apt_rootfs(rootfs)
+
+        apt_pkg.config.set('APT::Architecture', self.arch)
+        apt_pkg.config.set('Dir', rootfs)
+        apt_pkg.config.set('Dir::Cache', rootfs + '/var/cache/apt')
+        apt_pkg.config.set('Dir::State::status', rootfs + '/var/lib/dpkg/status')
+
+        apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
+
+    def isar_apt_update(self):
+        sources = apt_pkg.SourceList()
+        sources.read_main_list()
+
+        progress = apt.progress.text.AcquireProgress()
+
+        self.cache = apt_pkg.Cache()
+        self.cache.update(progress, sources)
+        self.cache = apt_pkg.Cache()
+
+    def bitbake(self, targets, task=''):
+        targets = [targets] if isinstance(targets, str) else targets
+        target_str = ''
+        for pn in targets:
+            if self.mc:
+                pn = 'mc:%s:%s' % (self.mc, pn)
+            target_str += '%s ' % (pn if not task else ':'.join([pn, task]))
+        targets.clear()
+        if target_str != '':
+            print('Building %s' % target_str)
+            return self.tinfoil.build_targets(target_str)
+        else:
+            return True
+
+    def deps_lookup(self, pkg, parent_provider, add=True):
+        if pkg in self.cache:
+            return
+
+        package_target = pkg if not self.mc else ':'.join(['mc', self.mc, pkg])
+
+        provider = self.tinfoil.find_best_provider(package_target)
+        if provider[3] is None or provider[3].endswith('linux-distro.bb'):
+            return
+
+        local_d = self.tinfoil.parse_recipe(package_target)
+        source_task = local_d.getVar('do_deploy_source', expand=True)
+
+        declared_deps = self.recipecache.deps[provider[3]]
+        build_deps = []
+
+        if hasattr(self, 'sr'):
+            self.sr.restart()
+
+            if self.sr.lookup(pkg):
+                if 'Build-Depends' in self.sr.build_depends:
+                    for dep in self.sr.build_depends['Build-Depends']:
+                        child = str(dep[0][0])
+                        if child not in build_deps:
+                            build_deps.append(child)
+            elif parent_provider and source_task:
+                self.need_source.append(pkg)
+        elif source_task:
+            self.need_source.append(pkg)
+
+        if parent_provider and add:
+            print(parent_provider[3] + ' depends on ' + pkg)
+            append_filename = parent_provider[3].rsplit(':', 1)[-1] + 'append'
+            with open(append_filename, 'a') as f:
+                f.write(f'DEPENDS += "{pkg}"\n\r')
+
+        for dep in declared_deps:
+            self.deps_lookup(dep, provider, False)
+        for dep in list(set(build_deps) - set(declared_deps)):
+            self.deps_lookup(dep, provider)
+
+    def init(self, mc, target):
+        self.mc = mc
+        self.target = target
+
+        package_target = target if not self.mc else ':'.join(['mc', self.mc, target])
+
+        self.d = self.tinfoil.parse_recipe(package_target)
+        self.distro = self.d.getVar('DISTRO', expand=True)
+        self.arch = self.d.getVar('DISTRO_ARCH', expand=True)
+
+        self.recipecache = self.tinfoil.cooker.recipecaches[mc]
+        self.apt_config()
+
+        self.need_source = []
+
+    def shutdown(self):
+        self.tinfoil.shutdown()
+
+    def explore_deps(self):
+        maxdepth = 10
+        while maxdepth > 0:
+            maxdepth -= 1
+
+            try:
+                self.isar_apt_update()
+                self.sr = apt_pkg.SourceRecords()
+            except:
+                print('No apt-cache found')
+
+            self.deps_lookup(self.target, None)
+
+            if not self.need_source:
+                break
+
+            print('Need sources for ' + str(self.need_source))
+
+            if not self.bitbake(self.need_source, 'do_deploy_source'):
+                break
+
+        if self.need_source:
+            print('Following packages still left unchecked: ' + str(self.need_source))
+
+if len(sys.argv) != 3:
+    print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base')
+    exit(1)
+
+mc = ''
+if sys.argv[2].startswith('mc:'):
+    args = sys.argv[2].split(':')
+    mc = args[1]
+    target = args[2]
+else:
+    args = sys.argv[2].split(':')
+    target = sys.argv[2]
+
+ib = IsarBuilder()
+try:
+    ib.init(mc, target)
+    ib.explore_deps()
+finally:
+    ib.shutdown()
-- 
2.34.1

[PATCH 3/3] isar: Add provides explorer

[Anton Mikanovich]

Explore Isar recipes for any packages declared in debian/control but
not added as PROVIDES of recipe.

Usage (inside build dir):
$ ../scripts/explore.py provides mc:qemuamd64-bullseye:isar-image-base

This script requires python3-apt to be installed.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 scripts/explore.py | 95 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 92 insertions(+), 3 deletions(-)

diff --git a/scripts/explore.py b/scripts/explore.py
index 56c7378a..ca387e6e 100755
--- a/scripts/explore.py
+++ b/scripts/explore.py
@@ -71,6 +71,89 @@ class IsarBuilder:
         else:
             return True
 
+    def provides_lookup(self):
+        try:
+            self.isar_apt_update()
+            self.sr = apt_pkg.SourceRecords()
+        except:
+            print('No apt-cache found')
+
+        sources_build = []
+        provides_all = []
+        for pn, fns in self.recipecache.pkg_pn.items():
+            if pn.endswith('-native') or pn.endswith('-compat'):
+                continue
+
+            provider = self.tinfoil.find_best_provider(pn)
+            if provider[3] is None or provider[3].endswith('linux-distro.bb'):
+                continue
+
+            pn_mc = pn if not self.mc else ':'.join(['mc', self.mc, pn])
+
+            local_d = self.tinfoil.parse_recipe(pn_mc)
+            source_task = local_d.getVar('do_deploy_source', expand=True)
+            if not source_task:
+                continue
+
+            # Filter out conflict packages
+            list = 0
+            provides = local_d.getVar('PROVIDES', expand=True)
+            for provides_list in provides_all:
+                for prov in provides.split():
+                    if prov in provides_list:
+                        break
+                else:
+                    break
+                list += 1
+
+            if list >= len(provides_all):
+                provides_all.append([])
+            if list >= len(sources_build):
+                sources_build.append([])
+
+            provides_all[list].extend(provides.split())
+            if hasattr(self, 'sr'):
+                self.sr.restart()
+                if not self.sr.lookup(pn):
+                    sources_build[list].append(pn)
+            else:
+                sources_build[list].append(pn)
+
+        for sources in sources_build:
+            if not self.bitbake(sources, 'do_deploy_source'):
+                return
+
+        if not hasattr(self, 'sr'):
+            try:
+                self.isar_apt_update()
+                self.sr = apt_pkg.SourceRecords()
+            except:
+                print('No apt-cache found')
+
+        for pn, fns in self.recipecache.pkg_pn.items():
+            if pn.endswith('-native') or pn.endswith('-compat'):
+                continue
+
+            provider = self.tinfoil.find_best_provider(pn)
+
+            self.sr.restart()
+            while self.sr.lookup(pn):
+                pn_mc = pn if not self.mc else ':'.join(['mc', self.mc, pn])
+                local_d = self.tinfoil.parse_recipe(pn_mc)
+                provides = local_d.getVar('PROVIDES', expand=True)
+
+                for bin in self.sr.binaries:
+                    if bin == pn:
+                        continue
+
+                    if bin in provides:
+                        continue
+
+                    print(f'{pn} provides {bin}')
+                    append_filename = provider[3].rsplit(':', 1)[-1] + 'append'
+                    with open(append_filename, 'a') as f:
+                        f.write(f'PROVIDES += "{bin}"\n')
+
     def deps_lookup(self, pkg, parent_provider, add=True):
         if pkg in self.cache:
             return
@@ -154,8 +237,8 @@ class IsarBuilder:
         if self.need_source:
             print('Following packages still left unchecked: ' + str(self.need_source))
 
-if len(sys.argv) != 3:
-    print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base')
+if len(sys.argv) != 3 or sys.argv[1] not in ['all', 'deps', 'provides']:
+    print('Usage: ../scripts/explore.py [ all | deps | provides ] mc:qemuamd64-bullseye:isar-image-base')
     exit(1)
 
 mc = ''
@@ -170,6 +253,12 @@ else:
 ib = IsarBuilder()
 try:
     ib.init(mc, target)
-    ib.explore_deps()
+    if sys.argv[1] == 'all':
+        ib.provides_lookup()
+        ib.explore_deps()
+    elif sys.argv[1] == 'deps':
+        ib.explore_deps()
+    elif sys.argv[1] == 'provides':
+        ib.provides_lookup()
 finally:
     ib.shutdown()
-- 
2.34.1