From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7281163473512300544 X-Received: by 2002:a05:6512:470:b0:4fd:d08c:fa3e with SMTP id x16-20020a056512047000b004fdd08cfa3emr4078851lfd.42.1695277981192; Wed, 20 Sep 2023 23:33:01 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a19:8c1c:0:b0:503:cdf:883a with SMTP id o28-20020a198c1c000000b005030cdf883als8918lfd.1.-pod-prod-01-eu; Wed, 20 Sep 2023 23:32:59 -0700 (PDT) X-Google-Smtp-Source: AGHT+IF2W2KawupNHw4AnN3bGjI3sswCgIBDwEb8LuyfKZAI0aMOXEBfbLZK/b26IadWyDfsc8DV X-Received: by 2002:a05:6512:3e03:b0:503:3644:4a98 with SMTP id i3-20020a0565123e0300b0050336444a98mr5396514lfv.2.1695277979208; Wed, 20 Sep 2023 23:32:59 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1695277979; cv=none; d=google.com; s=arc-20160816; b=GZgOLZmA8R6lHYosQAcwsimKVbPtN+2BEtNbXQgvFO08Z48qvkvXjcEuSmg+IakbBz Cu+SbXhyO8UAb/8yW9j92VTNTZb1PL9+ccF/aGZa9Btm+I3SrUOnL3PloWy5ufj5e2uZ U9xKfrW7ZYqu1pBL4OdE+Y6PKKuPQhulN8qmOYwyLnEi1wytitiyUSCs/bNAVVZ0c46S 0HTEnd2BAxkrU3GGS9FUzW9DHCmdkIrHIqvnYO+S6uHAXWj+asuh5hAJULeJmcnIh9zH MJ30IOhXkasjzrPPEoAX2qzgFEoEuMBr3bLSuJBkm8bklOL0YIcn56YkMz797Rl6jOye lFPA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=rspqLmmFraxDiHr+mvzbUSxbUHKLE9q6U716glJzMfc=; fh=/h9QQkzJ8EboVkWg45aWwpaUro6WMavIVd2OhN45RtE=; b=0z/qnWblC646/3xdg3m3K/ARujWPUgGhD0ab4NPOf915w1PP4Zq27SjWW8gmug0H0S kj69sC5XJFw0JKG4k/xgON12c8B4mneWI2q4rHE0xQZjw+090sJ+tl6Dn4RGS1DzxWLw qIBH7kiG0F8vaVm1Dqq4AVZVsciFhoIFyRcCQ1Z5LmOps4oF7dUJbD5VyBVeImli355M ZW/J6TAwUuIvfPzoHnq4EojBiqN9mxDyKOGSKtzdkGDUlU5tH9QwClyAVvKNDMCq7jqf 7sLkXrouD5aeASIEUgIAKTZG54lrS23pYeaYnlKw8UNMB3Mmk4AXsifXTIISjJM+lAAd BazA== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id b9-20020a056512070900b004fe3ba741c8si51979lfs.8.2023.09.20.23.32.59 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Wed, 20 Sep 2023 23:32:59 -0700 (PDT) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (r154-240-38-77-broadband.btv.lv [77.38.240.154] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPSA id 38L6Wt0i025667 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Thu, 21 Sep 2023 08:32:58 +0200 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH 2/3] isar: Add dependency explorer Date: Thu, 21 Sep 2023 09:32:46 +0300 Message-Id: <20230921063247.3114251-3-amikan@ilbers.de> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230921063247.3114251-1-amikan@ilbers.de> References: <20230921063247.3114251-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: 7+wfnN5a+xdz Explore Isar recipes for any build dependencies mentioned in source packages for not actually added at recipes. Usage (inside build dir): $ ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base After that normal Isar build can be performed. This script requires python3-apt to be installed. Signed-off-by: Anton Mikanovich --- scripts/explore.py | 175 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100755 scripts/explore.py diff --git a/scripts/explore.py b/scripts/explore.py new file mode 100755 index 00000000..56c7378a --- /dev/null +++ b/scripts/explore.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 + +import os +import sys +import apt_pkg +import apt.progress.base + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/../bitbake/lib') + +import bb.tinfoil + +class IsarBuilder: + def __init__(self): + self.tinfoil = bb.tinfoil.Tinfoil() + self.tinfoil.prepare() + + def create_apt_rootfs(self, rootfs): + repo_isar_dir = '%s/tmp/deploy/isar-apt/%s-%s/apt' % (os.path.abspath(os.getcwd()), self.distro, self.arch) + mirror = 'file://%s/%s' % (repo_isar_dir, self.distro) + codename = 'isar' + + if not os.path.exists(rootfs + '/var/lib/dpkg'): + os.makedirs(rootfs + '/var/lib/dpkg') + open(rootfs + '/var/lib/dpkg/status', 'w').close() + + if not os.path.exists(rootfs + '/etc/apt/preferences.d'): + os.makedirs(rootfs + '/etc/apt/preferences.d') + with open(rootfs + '/etc/apt/sources.list', 'w') as f: + f.write('deb [arch=%s] %s %s main\n' % (self.arch, mirror, codename)) + f.write('deb-src [arch=%s] %s %s main\n' % (self.arch, mirror, codename)) + + if not os.path.exists(rootfs + '/var/cache/apt/archives/partial'): + os.makedirs(rootfs + '/var/cache/apt/archives/partial') + + def apt_config(self): + apt_pkg.init() + + rootfs = os.path.abspath(os.getcwd()) + '/tmp/deps_poc_rootfs/%s-%s' % (self.distro, self.arch) + + if not os.path.isdir(rootfs): + self.create_apt_rootfs(rootfs) + + apt_pkg.config.set('APT::Architecture', self.arch) + apt_pkg.config.set('Dir', rootfs) + apt_pkg.config.set('Dir::Cache', rootfs + '/var/cache/apt') + apt_pkg.config.set('Dir::State::status', rootfs + '/var/lib/dpkg/status') + + apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1") + + def isar_apt_update(self): + sources = apt_pkg.SourceList() + sources.read_main_list() + + progress = apt.progress.text.AcquireProgress() + + self.cache = apt_pkg.Cache() + self.cache.update(progress, sources) + self.cache = apt_pkg.Cache() + + def bitbake(self, targets, task=''): + targets = [targets] if isinstance(targets, str) else targets + target_str = '' + for pn in targets: + if self.mc: + pn = 'mc:%s:%s' % (self.mc, pn) + target_str += '%s ' % (pn if not task else ':'.join([pn, task])) + targets.clear() + if target_str != '': + print('Building %s' % target_str) + return self.tinfoil.build_targets(target_str) + else: + return True + + def deps_lookup(self, pkg, parent_provider, add=True): + if pkg in self.cache: + return + + package_target = pkg if not self.mc else ':'.join(['mc', self.mc, pkg]) + + provider = self.tinfoil.find_best_provider(package_target) + if provider[3] is None or provider[3].endswith('linux-distro.bb'): + return + + local_d = self.tinfoil.parse_recipe(package_target) + source_task = local_d.getVar('do_deploy_source', expand=True) + + declared_deps = self.recipecache.deps[provider[3]] + build_deps = [] + + if hasattr(self, 'sr'): + self.sr.restart() + + if self.sr.lookup(pkg): + if 'Build-Depends' in self.sr.build_depends: + for dep in self.sr.build_depends['Build-Depends']: + child = str(dep[0][0]) + if child not in build_deps: + build_deps.append(child) + elif parent_provider and source_task: + self.need_source.append(pkg) + elif source_task: + self.need_source.append(pkg) + + if parent_provider and add: + print(parent_provider[3] + ' depends on ' + pkg) + append_filename = parent_provider[3].rsplit(':', 1)[-1] + 'append' + with open(append_filename, 'a') as f: + f.write(f'DEPENDS += "{pkg}"\n\r') + + for dep in declared_deps: + self.deps_lookup(dep, provider, False) + for dep in list(set(build_deps) - set(declared_deps)): + self.deps_lookup(dep, provider) + + def init(self, mc, target): + self.mc = mc + self.target = target + + package_target = target if not self.mc else ':'.join(['mc', self.mc, target]) + + self.d = self.tinfoil.parse_recipe(package_target) + self.distro = self.d.getVar('DISTRO', expand=True) + self.arch = self.d.getVar('DISTRO_ARCH', expand=True) + + self.recipecache = self.tinfoil.cooker.recipecaches[mc] + self.apt_config() + + self.need_source = [] + + def shutdown(self): + self.tinfoil.shutdown() + + def explore_deps(self): + maxdepth = 10 + while maxdepth > 0: + maxdepth -= 1 + + try: + self.isar_apt_update() + self.sr = apt_pkg.SourceRecords() + except: + print('No apt-cache found') + + self.deps_lookup(self.target, None) + + if not self.need_source: + break + + print('Need sources for ' + str(self.need_source)) + + if not self.bitbake(self.need_source, 'do_deploy_source'): + break + + if self.need_source: + print('Following packages still left unchecked: ' + str(self.need_source)) + +if len(sys.argv) != 3: + print('Usage: ../scripts/explore.py deps mc:qemuamd64-bullseye:isar-image-base') + exit(1) + +mc = '' +if sys.argv[2].startswith('mc:'): + args = sys.argv[2].split(':') + mc = args[1] + target = args[2] +else: + args = sys.argv[2].split(':') + target = sys.argv[2] + +ib = IsarBuilder() +try: + ib.init(mc, target) + ib.explore_deps() +finally: + ib.shutdown() -- 2.34.1