From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7068549159628832768 X-Received: by 2002:a5d:6f0b:0:b0:1ed:aac8:2853 with SMTP id ay11-20020a5d6f0b000000b001edaac82853mr5041790wrb.260.1645774858590; Thu, 24 Feb 2022 23:40:58 -0800 (PST) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:600c:1554:b0:37c:d848:5b20 with SMTP id f20-20020a05600c155400b0037cd8485b20ls34865wmg.1.gmail; Thu, 24 Feb 2022 23:40:57 -0800 (PST) X-Google-Smtp-Source: ABdhPJyotLG8KRrVs5cmimgu+XZCqjDw75fbtgH656f5wbfKyNgAbyMTRH0CYxgFry5+G7gaZ0Xe X-Received: by 2002:a5d:440c:0:b0:1ee:1c9a:193e with SMTP id z12-20020a5d440c000000b001ee1c9a193emr4252883wrq.523.1645774857643; Thu, 24 Feb 2022 23:40:57 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1645774857; cv=none; d=google.com; s=arc-20160816; b=SgwxfDQ68vg6QKVsC+nKQuFg6XEpaRdfkmgW4zms7+edX7ZH4NKrNEDUN9OJF1d+Vu Cf3DFDIOu0MrHqKBPITS24HkyNe25q9bkwUPYxQ23yuzgwl+WQ5nMn0fX77ccxB03pAd +LU2uAPxkIY6+a2o0XK1eXK7AmEJ4/vciNWpSF5qhHanS7Ij/pjzJqI9ec2bBsdG68l1 N/uQdwA81EWY2bLlCOTXA9Gu/spDkaiBBKm6d4aFWQloBSPRy/MUyKkr94Hb3xflyUy8 vOV8lXjBly3l4topsG993yI0GovN3o2L1b2/aUrvhGWHixkALs7dJcr4AOX7jKYQrNXn tqZw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=qR7otDF91wcDWnrpPms9Ewmm1uYiJtuKtFkbpC7nz/0=; b=kI+cFSGrEif8jQhMJYuDSLFlUC1IBZ/HoooUctLa/4MO/464jlCbenhctA4I6fzUbZ 1VWlC2ug8Qd9rEnoBjdSRcMLRAT8lnfAhcnAxCeVjhesD1CCjfxOUv93sYXjefS/jVxo WB3hVzAeWJhb+J7xmTl4lp9e9Uv/QHxmyBRTh1AJqV9R5z1daf1uF3g7SMhhhmAo+Z6+ FLQycBTBXzoIi8VlA0tEyo+PvuNg28hofPoFJ6m/e+lie8UClLU+pax9AzX+PdooOTKY aDuHZEaW74IcmFHPO0HObEazfrZYktXwqEmrNm0mHZWUkRdF6Q1U/9zOgAklvuH2r/xk 15mw== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id bg3-20020a05600c3c8300b0037e391f947bsi474809wmb.4.2022.02.24.23.40.57 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Thu, 24 Feb 2022 23:40:57 -0800 (PST) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from localhost.localdomain (mm-5-27-214-37.mgts.dynamic.pppoe.byfly.by [37.214.27.5] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 21P7emqt001558 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Fri, 25 Feb 2022 08:40:56 +0100 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [RFC 5/5] isar: Add external builder Date: Fri, 25 Feb 2022 10:40:40 +0300 Message-Id: <20220225074040.20975-6-amikan@ilbers.de> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220225074040.20975-1-amikan@ilbers.de> References: <20220225074040.20975-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: 36hAV1GAbqs5 Build Isar with external tool to parce all Debian package dependencies. Support only one target build for now, mc:qemuamd64-bullseye:isar-image-base is used by default. Usage (inside build dir): $ ../isar_builder.py Signed-off-by: Anton Mikanovich --- isar_builder.py | 142 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 isar_builder.py diff --git a/isar_builder.py b/isar_builder.py new file mode 100755 index 0000000..cf99c52 --- /dev/null +++ b/isar_builder.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +import os +import sys +import apt_pkg +import apt.progress.base + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/bitbake/lib') + +import bb.tinfoil + +class IsarBuilder: + def __init__(self): + self.tinfoil = bb.tinfoil.Tinfoil() + self.tinfoil.prepare() + + def create_apt_rootfs(self, rootfs): + repo_isar_dir = '%s/tmp/deploy/isar-apt/%s-%s/apt' % (os.path.abspath(os.getcwd()), self.distro, self.arch) + mirror = 'file://%s/%s' % (repo_isar_dir, self.distro) + codename = 'isar' + + if not os.path.exists(rootfs + '/var/lib/dpkg'): + os.makedirs(rootfs + '/var/lib/dpkg') + open(rootfs + '/var/lib/dpkg/status', 'w').close() + + if not os.path.exists(rootfs + '/etc/apt/preferences.d'): + os.makedirs(rootfs + '/etc/apt/preferences.d') + with open(rootfs + '/etc/apt/sources.list', 'w') as f: + f.write('deb [arch=%s] %s %s main\n' % (self.arch, mirror, codename)) + f.write('deb-src [arch=%s] %s %s main\n' % (self.arch, mirror, codename)) + + if not os.path.exists(rootfs + '/var/cache/apt/archives/partial'): + os.makedirs(rootfs + '/var/cache/apt/archives/partial') + + def apt_config(self): + apt_pkg.init() + + rootfs = os.path.abspath(os.getcwd()) + '/tmp/deps_poc_rootfs/%s-%s' % (self.distro, self.arch) + + if not os.path.isdir(rootfs): + self.create_apt_rootfs(rootfs) + + apt_pkg.config.set('APT::Architecture', self.arch) + apt_pkg.config.set('Dir', rootfs) + apt_pkg.config.set('Dir::Cache', rootfs + '/var/cache/apt') + apt_pkg.config.set('Dir::State::status', rootfs + '/var/lib/dpkg/status') + + apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1") + + def isar_apt_update(self): + sources = apt_pkg.SourceList() + sources.read_main_list() + + progress = apt.progress.text.AcquireProgress() + + self.cache = apt_pkg.Cache() + self.cache.update(progress, sources) + self.cache = apt_pkg.Cache() + + def srcpackage_lookup(self, pkg): + if pkg in self.cache: + return + if self.tinfoil.find_best_provider(pkg)[3] is None: + return + if pkg not in self.target_deps: + self.need_deb.append(pkg) + self.sr.restart() + if self.sr.lookup(pkg) is None: + self.need_source.append(pkg) + return + if 'Build-Depends' not in self.sr.build_depends: + return + for dep in self.sr.build_depends['Build-Depends']: + child = str(dep[0][0]) + if child not in self.checkdeps: + self.checkdeps.append(child) + self.srcpackage_lookup(child) + + def bitbake(self, mc, targets, task='', justbuild=[]): + targets = [targets] if isinstance(targets, str) else targets + target_str = '' + for pn in targets: + target_str += 'mc:%s:%s ' % (mc, pn if not task else ':'.join([pn, task])) + targets.clear() + for pn in justbuild: + target_str += 'mc:%s:%s ' % (mc, pn) + justbuild.clear() + if target_str != '': + print('Building %s' % target_str) + self.tinfoil.build_targets(target_str) + + def build(self, mc, target): + d = self.tinfoil.parse_recipe('multiconfig:%s:%s' % (mc, target)) + self.distro = d.getVar('DISTRO', expand=True) + self.arch = d.getVar('DISTRO_ARCH', expand=True) + + recipecache = self.tinfoil.cooker.recipecaches[mc] + provider = self.tinfoil.find_best_provider('multiconfig:%s:%s' % (mc, target)) + self.target_deps = recipecache.deps[provider[3]] + + kernel_image = d.getVar('KERNEL_IMAGE_PKG', expand=True) or '' + if kernel_image in self.target_deps: + kernel = self.tinfoil.find_best_provider('multiconfig:%s:%s' % (mc, kernel_image)) + if kernel[3].endswith('linux-distro.bb'): + self.target_deps.remove(kernel_image) + print('Remove %s from checking' % kernel_image) + + self.need_source = self.target_deps.copy() + self.need_deb = [] + + self.apt_config() + + maxdepth = 3 + while maxdepth > 0: + maxdepth -= 1 + + self.bitbake(mc, self.need_source, 'do_deploy_source', self.need_deb) + + self.isar_apt_update() + self.sr = apt_pkg.SourceRecords() + + self.checkdeps = [] + for dep in self.target_deps: + if dep not in self.checkdeps: + self.checkdeps.append(dep) + self.srcpackage_lookup(dep) + + if not self.need_source: + break + + if self.need_source: + print('Following packages still left unchecked: ' + str(self.need_source)) + + if self.need_deb: + print('Additional Debian dependencies found:' + str(self.need_deb)) + self.bitbake(mc, self.need_deb) + + # start final build + self.bitbake(mc, target) + +ib = IsarBuilder() +ib.build('qemuamd64-bullseye', 'isar-image-base') -- 2.25.1