* [PATCH 1/3] conf: Enable bbappends
2023-09-21 6:32 [PATCH 0/3] Introduce Debian dependencies and provides helper Anton Mikanovich
@ 2023-09-21 6:32 ` Anton Mikanovich
2023-09-21 6:32 ` [PATCH 2/3] isar: Add dependency explorer Anton Mikanovich
2023-09-21 6:32 ` [PATCH 3/3] isar: Add provides explorer Anton Mikanovich
2 siblings, 0 replies; 4+ messages in thread
From: Anton Mikanovich @ 2023-09-21 6:32 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Enable usage of bbappend files in all the layers.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta-isar/conf/layer.conf | 2 +-
meta-test/conf/layer.conf | 2 +-
meta/conf/layer.conf | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/meta-isar/conf/layer.conf b/meta-isar/conf/layer.conf
index dec2658f..2209b18b 100644
--- a/meta-isar/conf/layer.conf
+++ b/meta-isar/conf/layer.conf
@@ -5,7 +5,7 @@
BBPATH .= ":${LAYERDIR}"
# We have recipes-* directories, add to BBFILES
-BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
BBFILE_COLLECTIONS += "isar"
BBFILE_PATTERN_isar = "^${LAYERDIR}/"
diff --git a/meta-test/conf/layer.conf b/meta-test/conf/layer.conf
index 07b5c77d..0f15119d 100644
--- a/meta-test/conf/layer.conf
+++ b/meta-test/conf/layer.conf
@@ -5,7 +5,7 @@
BBPATH .= ":${LAYERDIR}"
# We have recipes-* directories, add to BBFILES
-BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
BBFILE_COLLECTIONS += "test"
BBFILE_PATTERN_test = "^${LAYERDIR}/"
diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index a38ab97d..b622ce69 100644
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -5,7 +5,7 @@
BBPATH .= ":${LAYERDIR}"
# We have recipes-* directories, add to BBFILES
-BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
BBFILE_COLLECTIONS += "core"
BBFILE_PATTERN_core = "^${LAYERDIR}/"
--
2.34.1
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH 2/3] isar: Add dependency explorer
2023-09-21 6:32 [PATCH 0/3] Introduce Debian dependencies and provides helper Anton Mikanovich
2023-09-21 6:32 ` [PATCH 1/3] conf: Enable bbappends Anton Mikanovich
@ 2023-09-21 6:32 ` Anton Mikanovich
2023-09-21 6:32 ` [PATCH 3/3] isar: Add provides explorer Anton Mikanovich
2 siblings, 0 replies; 4+ messages in thread
From: Anton Mikanovich @ 2023-09-21 6:32 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Explore Isar recipes for any build dependencies mentioned in source
packages for not actually added at recipes.
Usage (inside build dir):
$ ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base
After that normal Isar build can be performed.
This script requires python3-apt to be installed.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
scripts/explore.py | 175 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 175 insertions(+)
create mode 100755 scripts/explore.py
diff --git a/scripts/explore.py b/scripts/explore.py
new file mode 100755
index 00000000..56c7378a
--- /dev/null
+++ b/scripts/explore.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import apt_pkg
+import apt.progress.base
+
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/../bitbake/lib')
+
+import bb.tinfoil
+
+class IsarBuilder:
+ def __init__(self):
+ self.tinfoil = bb.tinfoil.Tinfoil()
+ self.tinfoil.prepare()
+
+ def create_apt_rootfs(self, rootfs):
+ repo_isar_dir = '%s/tmp/deploy/isar-apt/%s-%s/apt' % (os.path.abspath(os.getcwd()), self.distro, self.arch)
+ mirror = 'file://%s/%s' % (repo_isar_dir, self.distro)
+ codename = 'isar'
+
+ if not os.path.exists(rootfs + '/var/lib/dpkg'):
+ os.makedirs(rootfs + '/var/lib/dpkg')
+ open(rootfs + '/var/lib/dpkg/status', 'w').close()
+
+ if not os.path.exists(rootfs + '/etc/apt/preferences.d'):
+ os.makedirs(rootfs + '/etc/apt/preferences.d')
+ with open(rootfs + '/etc/apt/sources.list', 'w') as f:
+ f.write('deb [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+ f.write('deb-src [arch=%s] %s %s main\n' % (self.arch, mirror, codename))
+
+ if not os.path.exists(rootfs + '/var/cache/apt/archives/partial'):
+ os.makedirs(rootfs + '/var/cache/apt/archives/partial')
+
+ def apt_config(self):
+ apt_pkg.init()
+
+ rootfs = os.path.abspath(os.getcwd()) + '/tmp/deps_poc_rootfs/%s-%s' % (self.distro, self.arch)
+
+ if not os.path.isdir(rootfs):
+ self.create_apt_rootfs(rootfs)
+
+ apt_pkg.config.set('APT::Architecture', self.arch)
+ apt_pkg.config.set('Dir', rootfs)
+ apt_pkg.config.set('Dir::Cache', rootfs + '/var/cache/apt')
+ apt_pkg.config.set('Dir::State::status', rootfs + '/var/lib/dpkg/status')
+
+ apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
+
+ def isar_apt_update(self):
+ sources = apt_pkg.SourceList()
+ sources.read_main_list()
+
+ progress = apt.progress.text.AcquireProgress()
+
+ self.cache = apt_pkg.Cache()
+ self.cache.update(progress, sources)
+ self.cache = apt_pkg.Cache()
+
+ def bitbake(self, targets, task=''):
+ targets = [targets] if isinstance(targets, str) else targets
+ target_str = ''
+ for pn in targets:
+ if self.mc:
+ pn = 'mc:%s:%s' % (self.mc, pn)
+ target_str += '%s ' % (pn if not task else ':'.join([pn, task]))
+ targets.clear()
+ if target_str != '':
+ print('Building %s' % target_str)
+ return self.tinfoil.build_targets(target_str)
+ else:
+ return True
+
+ def deps_lookup(self, pkg, parent_provider, add=True):
+ if pkg in self.cache:
+ return
+
+ package_target = pkg if not self.mc else ':'.join(['mc', self.mc, pkg])
+
+ provider = self.tinfoil.find_best_provider(package_target)
+ if provider[3] is None or provider[3].endswith('linux-distro.bb'):
+ return
+
+ local_d = self.tinfoil.parse_recipe(package_target)
+ source_task = local_d.getVar('do_deploy_source', expand=True)
+
+ declared_deps = self.recipecache.deps[provider[3]]
+ build_deps = []
+
+ if hasattr(self, 'sr'):
+ self.sr.restart()
+
+ if self.sr.lookup(pkg):
+ if 'Build-Depends' in self.sr.build_depends:
+ for dep in self.sr.build_depends['Build-Depends']:
+ child = str(dep[0][0])
+ if child not in build_deps:
+ build_deps.append(child)
+ elif parent_provider and source_task:
+ self.need_source.append(pkg)
+ elif source_task:
+ self.need_source.append(pkg)
+
+ if parent_provider and add:
+ print(parent_provider[3] + ' depends on ' + pkg)
+ append_filename = parent_provider[3].rsplit(':', 1)[-1] + 'append'
+ with open(append_filename, 'a') as f:
+ f.write(f'DEPENDS += "{pkg}"\n\r')
+
+ for dep in declared_deps:
+ self.deps_lookup(dep, provider, False)
+ for dep in list(set(build_deps) - set(declared_deps)):
+ self.deps_lookup(dep, provider)
+
+ def init(self, mc, target):
+ self.mc = mc
+ self.target = target
+
+ package_target = target if not self.mc else ':'.join(['mc', self.mc, target])
+
+ self.d = self.tinfoil.parse_recipe(package_target)
+ self.distro = self.d.getVar('DISTRO', expand=True)
+ self.arch = self.d.getVar('DISTRO_ARCH', expand=True)
+
+ self.recipecache = self.tinfoil.cooker.recipecaches[mc]
+ self.apt_config()
+
+ self.need_source = []
+
+ def shutdown(self):
+ self.tinfoil.shutdown()
+
+ def explore_deps(self):
+ maxdepth = 10
+ while maxdepth > 0:
+ maxdepth -= 1
+
+ try:
+ self.isar_apt_update()
+ self.sr = apt_pkg.SourceRecords()
+ except:
+ print('No apt-cache found')
+
+ self.deps_lookup(self.target, None)
+
+ if not self.need_source:
+ break
+
+ print('Need sources for ' + str(self.need_source))
+
+ if not self.bitbake(self.need_source, 'do_deploy_source'):
+ break
+
+ if self.need_source:
+ print('Following packages still left unchecked: ' + str(self.need_source))
+
+if len(sys.argv) != 3:
+ print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base')
+ exit(1)
+
+mc = ''
+if sys.argv[2].startswith('mc:'):
+ args = sys.argv[2].split(':')
+ mc = args[1]
+ target = args[2]
+else:
+ args = sys.argv[2].split(':')
+ target = sys.argv[2]
+
+ib = IsarBuilder()
+try:
+ ib.init(mc, target)
+ ib.explore_deps()
+finally:
+ ib.shutdown()
--
2.34.1
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH 3/3] isar: Add provides explorer
2023-09-21 6:32 [PATCH 0/3] Introduce Debian dependencies and provides helper Anton Mikanovich
2023-09-21 6:32 ` [PATCH 1/3] conf: Enable bbappends Anton Mikanovich
2023-09-21 6:32 ` [PATCH 2/3] isar: Add dependency explorer Anton Mikanovich
@ 2023-09-21 6:32 ` Anton Mikanovich
2 siblings, 0 replies; 4+ messages in thread
From: Anton Mikanovich @ 2023-09-21 6:32 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Explore Isar recipes for any packages declared in debian/control but
not added as PROVIDES of recipe.
Usage (inside build dir):
$ ../scripts/explore.py provides mc:qemuamd64-bullseye:isar-image-base
This script requires python3-apt to be installed.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
scripts/explore.py | 95 ++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 92 insertions(+), 3 deletions(-)
diff --git a/scripts/explore.py b/scripts/explore.py
index 56c7378a..ca387e6e 100755
--- a/scripts/explore.py
+++ b/scripts/explore.py
@@ -71,6 +71,89 @@ class IsarBuilder:
else:
return True
+ def provides_lookup(self):
+ try:
+ self.isar_apt_update()
+ self.sr = apt_pkg.SourceRecords()
+ except:
+ print('No apt-cache found')
+
+ sources_build = []
+ provides_all = []
+ for pn, fns in self.recipecache.pkg_pn.items():
+ if pn.endswith('-native') or pn.endswith('-compat'):
+ continue
+
+ provider = self.tinfoil.find_best_provider(pn)
+ if provider[3] is None or provider[3].endswith('linux-distro.bb'):
+ continue
+
+ pn_mc = pn if not self.mc else ':'.join(['mc', self.mc, pn])
+
+ local_d = self.tinfoil.parse_recipe(pn_mc)
+ source_task = local_d.getVar('do_deploy_source', expand=True)
+ if not source_task:
+ continue
+
+ # Filter out conflict packages
+ list = 0
+ provides = local_d.getVar('PROVIDES', expand=True)
+ for provides_list in provides_all:
+ for prov in provides.split():
+ if prov in provides_list:
+ break
+ else:
+ break
+ list += 1
+
+ if list >= len(provides_all):
+ provides_all.append([])
+ if list >= len(sources_build):
+ sources_build.append([])
+
+ provides_all[list].extend(provides.split())
+ if hasattr(self, 'sr'):
+ self.sr.restart()
+ if not self.sr.lookup(pn):
+ sources_build[list].append(pn)
+ else:
+ sources_build[list].append(pn)
+
+ for sources in sources_build:
+ if not self.bitbake(sources, 'do_deploy_source'):
+ return
+
+ if not hasattr(self, 'sr'):
+ try:
+ self.isar_apt_update()
+ self.sr = apt_pkg.SourceRecords()
+ except:
+ print('No apt-cache found')
+
+ for pn, fns in self.recipecache.pkg_pn.items():
+ if pn.endswith('-native') or pn.endswith('-compat'):
+ continue
+
+ provider = self.tinfoil.find_best_provider(pn)
+
+ self.sr.restart()
+ while self.sr.lookup(pn):
+ pn_mc = pn if not self.mc else ':'.join(['mc', self.mc, pn])
+ local_d = self.tinfoil.parse_recipe(pn_mc)
+ provides = local_d.getVar('PROVIDES', expand=True)
+
+ for bin in self.sr.binaries:
+ if bin == pn:
+ continue
+
+ if bin in provides:
+ continue
+
+ print(f'{pn} provides {bin}')
+ append_filename = provider[3].rsplit(':', 1)[-1] + 'append'
+ with open(append_filename, 'a') as f:
+ f.write(f'PROVIDES += "{bin}"\n')
+
def deps_lookup(self, pkg, parent_provider, add=True):
if pkg in self.cache:
return
@@ -154,8 +237,8 @@ class IsarBuilder:
if self.need_source:
print('Following packages still left unchecked: ' + str(self.need_source))
-if len(sys.argv) != 3:
- print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base')
+if len(sys.argv) != 3 or sys.argv[1] not in ['all', 'deps', 'provides']:
+ print('Usage: ../scripts/explore.py [ all | deps | provides ] mc:qemuamd64-bullseye:isar-image-base')
exit(1)
mc = ''
@@ -170,6 +253,12 @@ else:
ib = IsarBuilder()
try:
ib.init(mc, target)
- ib.explore_deps()
+ if sys.argv[1] == 'all':
+ ib.provides_lookup()
+ ib.explore_deps()
+ elif sys.argv[1] == 'deps':
+ ib.explore_deps()
+ elif sys.argv[1] == 'provides':
+ ib.provides_lookup()
finally:
ib.shutdown()
--
2.34.1
^ permalink raw reply [flat|nested] 4+ messages in thread