From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7078983779658760192 X-Received: by 2002:a19:710a:0:b0:443:ad21:dcc0 with SMTP id m10-20020a19710a000000b00443ad21dcc0mr7337738lfc.688.1648204351208; Fri, 25 Mar 2022 03:32:31 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a05:6512:b05:b0:448:696a:ea62 with SMTP id w5-20020a0565120b0500b00448696aea62ls212903lfu.0.gmail; Fri, 25 Mar 2022 03:32:29 -0700 (PDT) X-Google-Smtp-Source: ABdhPJxbOqsPcqFkFM+jb8pKxY6rSeqVgt+XpEhpSEZcfkjbLe1nlFCR8twbZVKVw13sWzSMRq8r X-Received: by 2002:a05:6512:3e28:b0:44a:126b:fecb with SMTP id i40-20020a0565123e2800b0044a126bfecbmr6991446lfv.423.1648204349828; Fri, 25 Mar 2022 03:32:29 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1648204349; cv=none; d=google.com; s=arc-20160816; b=IcTFoUdu4LOOeDonVPV4/SsU0xxQwDvsWnsQjBSs8aWm1H+D7UYYhjOwf70XT100nE x4PdL90s079L7/jHTbltosabuVjKRPFCCZnw1D+q7ViPcuCtlXj3bqjw6t1Mm3W2GOtc hDUS+gq3IXsJFoTsve3Y578HFPfOAxNZykUDiwEMQMAamuCriq0IB91pKX3hfyqfUrs3 Md7cYTkkBG+BFJkXVQ4aDbtpH2uBL/L6vRfMKRynaNZCb4ZD1DWPHM9hYHRVLvQSzS+6 M8WHhtbmI2asI0UxjM8oZtZ1CApEhM3ObisteTw6R+iiNbnok5ELOjC2F6wnUD/Er9hg eLRA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from; bh=d9F9ktOdP2bImB90RnkwRWjCROwUHN4Ceu+xDlM2f6M=; b=RhtGsyIvj5eZbOKFA5v5qQwF9CAXtbSqBVmA/cgXtFHdKl2k5ndv414gb8WlVWnHE1 IEzzoAOzUCuSqSTh1uB+LXC9YISPabc7iAPfqYAAS8CPEJ463yokBfToF2SSmat3C1nV reDrccygLi8xMSWxsZ8h5WHWex+D7NfSyGw3wLMSWkG2jQ1T6v3VqDP7NbQhWLP7HY0Y ZKy9tYzWD92rHWv8Zn/2YS1wOhGK25q5T4yAwLx7dIMY1wv4gmYq7PnFSiQpU9aQ6YnQ BK3XXQsSNzP507kT73ZadYlVZDCOX+Qb76/FUKkg6Ay4D/OeP1wfaeOZRY/AFIhSwaH6 6S7w== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id f2-20020a2e1f02000000b00244ddef9705si269981ljf.1.2022.03.25.03.32.29 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Fri, 25 Mar 2022 03:32:29 -0700 (PDT) Received-SPF: pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Received: from baighyz.m.ilbers.de (host-80-81-17-52.static.customer.m-online.net [80.81.17.52]) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8) with ESMTPSA id 22PAWQZX030611 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT) for ; Fri, 25 Mar 2022 11:32:27 +0100 From: Uladzimir Bely To: isar-users@googlegroups.com Subject: [PATCH v3 1/5] Add debrepo python script handling base-apt Date: Fri, 25 Mar 2022 11:32:22 +0100 Message-Id: <20220325103226.27033-2-ubely@ilbers.de> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20220325103226.27033-1-ubely@ilbers.de> References: <20220325103226.27033-1-ubely@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: 7bnj/Svk53K8 Signed-off-by: Uladzimir Bely --- scripts/debrepo | 361 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100755 scripts/debrepo diff --git a/scripts/debrepo b/scripts/debrepo new file mode 100755 index 00000000..fba8342e --- /dev/null +++ b/scripts/debrepo @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 + +# This software is a part of ISAR. +# Copyright (C) 2022 ilbers GmbH + +import os +import sys + +import subprocess +import getopt +import pickle + +import apt_pkg +import apt.progress.base + + +class DebRepo(object): + def __init__(self, workdir, cmdline_opts): + self.workdir = workdir + self.optsfile = self.workdir + "/repo.opts" + + # Set default values + self.distro = "debian" + + self.repo = self.workdir + "/repo/apt" + "/" + self.distro + self.repodb = self.workdir + "/repo/db" + "/" + self.distro + self.mirror = "http://deb.debian.org/debian" + self.arch = "amd64" + self.codename = "bullseye" + self.components = "main contrib non-free" + self.keydir = "/etc/apt/trusted.gpg.d" + self.check_gpg = True + + # Load stored opts + opts = self.load_opts() + print("stored opts: " + str(opts)) + + # Overwrite opts by cmdline_opts + for opt, arg in cmdline_opts.items(): + opts[opt] = arg + + print("all opts: " + str(opts)) + + # Replace by values passed in commandline + for opt, arg in opts.items(): + if opt == "repodir": + self.repo = arg + "/" + self.distro + if opt == "repodbdir": + self.repodb = arg + "/" + self.distro + if opt == "mirror": + self.mirror = arg + if opt == "arch": + self.arch = arg + if opt == "distro": + self.distro = arg + if opt == "codename": + self.codename = arg + if opt == "components": + self.components = arg.replace(",", " ") + if opt == "keydir": + self.keydir = arg + if opt == "check_gpg": + self.check_gpg = arg + + self.save_opts(opts) + + print("workdir: " + str(self.workdir)) + print("repo: " + str(self.repo)) + print("repodb: " + str(self.repodb)) + print("mirror: " + str(self.mirror)) + print("arch: " + str(self.arch)) + print("distro: " + str(self.distro)) + print("codename: " + str(self.codename)) + print("components: " + str(self.components)) + print("keydir: " + str(self.keydir)) + print("check_gpg: " + str(self.check_gpg)) + + self.cache = None + self.depcache = None + self.sr = None + + def create_rootfs(self): + if not os.path.exists(self.workdir + "/var/lib/dpkg"): + os.makedirs(self.workdir + "/var/lib/dpkg") + f = open(self.workdir + "/var/lib/dpkg" + "/status", "w") + f.flush + f.close + + if not os.path.exists(self.workdir + "/etc/apt/"): + os.makedirs(self.workdir + "/etc/apt/") + f = open(self.workdir + "/etc/apt/sources.list", "w") + f.write("deb [arch=" + self.arch + "] " + self.mirror + " " + self.codename + " " + self.components + "\n") + f.write("deb-src [arch=" + self.arch + "] " + self.mirror + " " + self.codename + " " + self.components + "\n") + f.flush + f.close + + if not os.path.exists(self.workdir + "/var/cache/apt/archives/partial"): + os.makedirs(self.workdir + "/var/cache/apt/archives/partial") + + def create_repo_dist(self): + if not os.path.exists(self.repo + "/conf"): + os.makedirs(self.repo + "/conf") + f = open(self.repo + "/conf" + "/distributions", "w") + f.write("Codename: " + self.codename + "\n") + f.write("Architectures: i386 armhf arm64 amd64 mipsel riscv64 source" + "\n") + f.write("Components: " + self.components + "\n") + f.flush + f.close + + def apt_config(self): + # Configure apt to work with empty directory + apt_pkg.config.set("APT::Architecture", self.arch) + apt_pkg.config.set("Dir", self.workdir) + apt_pkg.config.set("Dir::Cache", self.workdir + "/var/cache/apt") + apt_pkg.config.set("Dir::State::status", self.workdir + "/var/lib/dpkg/status") + + apt_pkg.config.set("APT::Install-Recommends", "0") + apt_pkg.config.set("APT::Install-Suggests", "0") + + # Use host keys for authentification + # apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg") + # apt_pkg.config.set("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d") + apt_pkg.config.set("Dir::Etc::TrustedParts", self.keydir) + + # Allow using repositories without keys + if not self.check_gpg: + apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1") + + def mark_essential(self): + for pkg in self.cache.packages: + if pkg.essential: + self.depcache.mark_install(pkg) + + def mark_by_prio(self, priority): + for pkg in self.cache.packages: + ver = self.depcache.get_candidate_ver(pkg) + if ver and ver.priority <= priority: + self.depcache.mark_install(pkg) + + def mark_list(self, pkglist): + if pkglist: + for pkgname in pkglist: + if pkgname in self.cache: + pkg = self.cache[pkgname] + self.depcache.mark_install(pkg) + + def handle_deb(self, item): + subprocess.run([ + "reprepro", + "--dbdir", self.repodb, + "--outdir", self.repo, + "--confdir", self.repo + "/conf", + "-C", "main", + "includedeb", + self.codename, + item.destfile + ]) + + def handle_repo(self, fetcher): + fetcher.run() + for item in fetcher.items: + if item.status == item.STAT_ERROR: + print("Some error ocured: '%s'" % item.error_text) + pass + else: + self.handle_deb(item) + + def get_filename(self, url): + fragment_removed = url.split("#")[0] # keep to left of first # + query_string_removed = fragment_removed.split("?")[0] + scheme_removed = query_string_removed.split("://")[-1].split(":")[-1] + if scheme_removed.find("/") == -1: + return "" + return os.path.basename(scheme_removed) + + def download_file(self, uri): + filename = self.get_filename(uri) + subprocess.run([ + "wget", + uri, + "-O", + filename + ]) + + def handle_dsc(self, uri): + filename = self.get_filename(uri) + subprocess.run([ + "reprepro", + "--dbdir", self.repodb, + "--outdir", self.repo, + "--confdir", self.repo + "/conf", + "-C", "main", + "-S", "-", "-P" "source", + "--delete", + "includedsc", + self.codename, + os.path.realpath(filename) + ]) + + def handle_src_list(self, pkgs): + if pkgs: + for pkgname in pkgs: + dsc = "" + self.sr.restart() + if self.sr.lookup(pkgname): + print(self.sr.files) + for sr_file in self.sr.files: + print(self.sr.index.archive_uri(sr_file[2])) + self.download_file(self.sr.index.archive_uri(sr_file[2])) + + dsc_uri = self.sr.index.archive_uri(self.sr.files[0][2]) + self.handle_dsc(dsc_uri) + + def apt_run(self, init, srcmode, pkgs, controlfile): + apt_pkg.init() + + sources = apt_pkg.SourceList() + sources.read_main_list() + + progress = apt.progress.text.AcquireProgress() + + self.cache = apt_pkg.Cache() + self.cache.update(progress, sources) + self.cache = apt_pkg.Cache() + + self.depcache = apt_pkg.DepCache(self.cache) + self.sr = apt_pkg.SourceRecords() + + if init: + self.mark_essential() + # 1(required), 2(important), 3(standard), 4(optional), 5(extra) + self.mark_by_prio(1) + + if srcmode: + self.handle_src_list(pkgs) + else: + self.mark_list(pkgs) + + if controlfile: + fobj = open(controlfile, "r") + try: + Parse = apt_pkg.TagFile(fobj) + while Parse.step() == 1: + for item in apt_pkg.parse_depends(Parse.section.get("Build-Depends", "")): + #self.mark_list([item[0][0]]) + if item[0][0] in self.cache: + pkg = self.cache[item[0][0]] + print("pkg: " + str(pkg)) + self.depcache.mark_install(pkg) + + finally: + fobj.close() + + if init or not srcmode: + fetcher = apt_pkg.Acquire(progress) + pm = apt_pkg.PackageManager(self.depcache) + + recs = apt_pkg.PackageRecords(self.cache) + pm.get_archives(fetcher, sources, recs) + + self.handle_repo(fetcher) + + def load_opts(self): + params = {} + if os.path.isfile(self.optsfile): + with open(self.optsfile, 'rb') as file: + data = file.read() + if data: + params = pickle.loads(data) + + return params + + def save_opts(self, opts): + file = open(self.optsfile, 'wb') + pickle.dump(opts, file) + file.close() + + +class DebRepoArgs(object): + def __init__(self): + self.workdir = "" + self.init = False + self.srcmode = False + self.controlfile = "" + + self.opts = {} + self.pkgs = [] + + try: + opts, args = getopt.getopt(sys.argv[1:], "", [ + "init", + "srcmode", + "workdir=", + "repodir=", + "repodbdir=", + "mirror=", + "arch=", + "distro=", + "codename=", + "components=", + "keydir=", + "no-check-gpg", + "controlfile=", + ]) + except getopt.GetoptError as msg: + print("Error: " + str(msg)) + sys.exit(1) + + for opt, arg in opts: + if opt in ("--workdir"): + self.workdir = arg + if opt in ("--init"): + self.init = True + if opt in ("--srcmode"): + self.srcmode = True + if opt in ("--controlfile"): + self.controlfile = arg + + if opt in ("--repodir", + "--repodbdir", + "--mirror", + "--arch", + "--distro", + "--codename", + "--components", + "--keydir", + "--controlfile", + ): + self.opts[opt[2:]] = arg + if opt in ("--no-check-gpg"): + self.opts['check_gpg'] = False + + if not self.workdir: + print("Error: workdir is not specified") + sys.exit(1) + + self.pkgs = args + + +def main(): + args = DebRepoArgs() + + if not (args.init or args.pkgs or args.controlfile): + print("Nothing to do") + sys.exit(0) + + if not os.path.exists(args.workdir): + os.makedirs(args.workdir) + + debrepo = DebRepo(args.workdir, args.opts) + + if (args.init): + debrepo.create_rootfs() + debrepo.create_repo_dist() + + debrepo.apt_config() + debrepo.apt_run(args.init, args.srcmode, args.pkgs, args.controlfile) + + +if __name__ == "__main__": + main() -- 2.20.1