From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7202866696458076160 X-Received: by 2002:a5b:7:0:b0:802:5b06:4d1d with SMTP id a7-20020a5b0007000000b008025b064d1dmr103384ybp.527.1677048092555; Tue, 21 Feb 2023 22:41:32 -0800 (PST) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a81:f43:0:b0:533:95cb:662a with SMTP id 64-20020a810f43000000b0053395cb662als3211680ywp.10.-pod-prod-gmail; Tue, 21 Feb 2023 22:41:31 -0800 (PST) X-Google-Smtp-Source: AK7set9xQzRQ8ditjUopqxESHt5G8YJnMysXlGPQ1qyNU9FQ3nqiErGSUW4GcZ6mL8QFe/hbyfDZ X-Received: by 2002:a0d:c8c2:0:b0:3b9:d766:dd70 with SMTP id k185-20020a0dc8c2000000b003b9d766dd70mr5692444ywd.44.1677048091895; Tue, 21 Feb 2023 22:41:31 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1677048091; cv=none; d=google.com; s=arc-20160816; b=nCwB/rFxZhoAUMqoWWvNPFmHPIwx40j/Kph64mNohsiXBp3wGQYKpcvSjM/fWJdKis FNSZ7kTLoYvgKLiUnrG/uDCdas9l5tseSyTnQlfApuh2I2935IVAo7WOdD+EFjrq2235 jj/fa5UzhmHEAVPWeEusykEsznLkyr5nMArzzha2ELD+NvSu7EoBZX4TJYFPEzD3JDWI suXQSyi5hQA+w/UWSYi7qCFsHDP853DSBjD5fn9j2ULV/Baoq0TrRkRZT3WpLGxTL69L Wy/XO/o7Hl9KPX9u24ROk58sejQVPGMLDFAd6bYim5g3qDCokAAQruEA1S0IJ9+z3GBB EpdQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from; bh=bKs2nIWAMFtm0tPbJIbH+1KuwE0z6w9QpylsPXYFl/M=; b=nMSFA8Szdq400voSZ4byFeH+ZeE3/5pSMV0FwmpOeqD7GRd51rAJnEwrNfOxOiwl20 bArxObQmlw7Vmc0fciLJNCq+sXn0bCw9JV+JUGk55wN92LJQPrg8MENnO8x0sSni5nbD 5BVZtxgHySMZ2iW57XdmYC0lA/jhxwm+K9Iyryn7CCSO0kHdELeFxhbIJFUMKrtESthr I6URr9DecC8e/l53mtJAkOJByFy378AQBEMeLgdrZp86AVtuvkGj4t5iVsCZ8HhlhC40 yiokanQMNvwFy0bxo2ZBde60kXTarhYvz2gJIcAKKfAr4++jNGXbkVzErlm2zCp3EWw0 aiOQ== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id j135-20020a81558d000000b00533bad9d28csi580556ywb.4.2023.02.21.22.41.31 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Tue, 21 Feb 2023 22:41:31 -0800 (PST) Received-SPF: pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of ubely@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=ubely@ilbers.de Received: from baighyz.m.ilbers.de (host-80-81-17-52.static.customer.m-online.net [80.81.17.52]) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPSA id 31M6fPcE011872 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT) for ; Wed, 22 Feb 2023 07:41:29 +0100 From: Uladzimir Bely To: isar-users@googlegroups.com Subject: [PATCH v4 10/13] base-apt: Predownload packages to base-apt before install Date: Wed, 22 Feb 2023 07:41:22 +0100 Message-Id: <20230222064125.22754-11-ubely@ilbers.de> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20230222064125.22754-1-ubely@ilbers.de> References: <20230222064125.22754-1-ubely@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: cXOLUCmnwRvR This patch uses debrepo script to predownload packages to base-apt repository before they are installed in buildchroot/rootfs. Signed-off-by: Uladzimir Bely --- meta/classes/dpkg-base.bbclass | 31 ++++++++++++++++++-- meta/classes/dpkg.bbclass | 4 ++- meta/classes/image-locales-extension.bbclass | 5 ++++ meta/classes/image-tools-extension.bbclass | 9 ++++++ meta/classes/rootfs.bbclass | 4 +++ 5 files changed, 50 insertions(+), 3 deletions(-) diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass index eb4ac34a..ee745ee1 100644 --- a/meta/classes/dpkg-base.bbclass +++ b/meta/classes/dpkg-base.bbclass @@ -21,6 +21,8 @@ python __anonymous() { d.setVar('DEBREPO_WORKDIR', d.getVar('DEBREPO_HOST_DIR', True)) } +inherit debrepo + DEPENDS ?= "" RPROVIDES ?= "${PROVIDES}" @@ -118,16 +120,30 @@ do_apt_fetch() { E="${@ isar_export_proxies(d)}" schroot_create_configs + debrepo_add_packages --srcmode "${DEBREPO_TARGET_DIR}" "${SRC_APT}" + schroot_cleanup() { schroot_delete_configs } trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT + # Begin chroot session + session_id=$(schroot -b -c ${SBUILD_CHROOT}) + + flock -x "${REPO_BASE_DIR}/repo.lock" -c " + schroot -d / -r -c ${session_id} -u root -- \ + sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" ' + " + for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -d / -r -c ${session_id} -- \ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done + + # End chroot session + schroot -e -c ${session_id} + schroot_delete_configs } @@ -148,8 +164,16 @@ do_apt_unpack() { trap 'exit 1' INT HUP QUIT TERM ALRM USR1 trap 'schroot_cleanup' EXIT + # Begin chroot session + session_id=$(schroot -b -c ${SBUILD_CHROOT}) + + flock -x "${REPO_BASE_DIR}/repo.lock" -c " + schroot -d / -r -c ${session_id} -u root -- \ + sh -c 'apt-get -y update -o Dir::Etc::SourceList=\"sources.list.d/base-apt.list\" -o Dir::Etc::SourceParts=\"-\" ' + " + for uri in "${SRC_APT}"; do - schroot -d / -c ${SBUILD_CHROOT} -- \ + schroot -d / -r -c ${session_id} -- \ sh -c ' \ set -e dscfile="$(apt-get -y -qq --print-uris --only-source source "${2}" | cut -d " " -f2 | grep -E "*.dsc")" @@ -158,6 +182,9 @@ do_apt_unpack() { dpkg-source -x "${dscfile}" "${PPS}"' \ my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${uri}" done + + # End chroot session + schroot -e -c ${session_id} schroot_delete_configs } do_apt_unpack[network] = "${TASK_USE_SUDO}" diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass index d5285032..b3d03516 100644 --- a/meta/classes/dpkg.bbclass +++ b/meta/classes/dpkg.bbclass @@ -32,6 +32,7 @@ dpkg_runbuild() { export PARALLEL_MAKE="${PARALLEL_MAKE}" rm -f ${SBUILD_CONFIG} + debrepo_handle_controlfile "${WORKDIR}/${PPS}/debian/control" env | while read -r line; do # Filter the same lines @@ -104,11 +105,12 @@ dpkg_runbuild() { sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \ --host=${PACKAGE_ARCH} --build=${SBUILD_HOST_ARCH} ${profiles} \ --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \ - --no-apt-update \ + --no-apt-update --no-apt-upgrade --no-apt-distupgrade \ --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \ --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \ --chroot-setup-commands="rm -f /var/log/dpkg.log" \ --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/ || :" \ + --chroot-setup-commands="flock -x /base-apt/repo.lock -c 'apt-get -y update'" \ --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \ --finished-build-commands="cp -Ln --no-preserve=owner ${deb_dir}/*.deb -t ${ext_deb_dir}/ || :" \ --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \ diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass index 65b9ac80..4bdcb39d 100644 --- a/meta/classes/image-locales-extension.bbclass +++ b/meta/classes/image-locales-extension.bbclass @@ -6,6 +6,8 @@ # This class extends the image.bbclass for setting locales and purging unneeded # ones. +inherit debrepo + LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\ en_US ISO-8859-1\n" LOCALE_DEFAULT ?= "en_US.UTF-8" @@ -29,6 +31,9 @@ ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download" image_install_localepurge_download[weight] = "40" image_install_localepurge_download[network] = "${TASK_USE_NETWORK_AND_SUDO}" image_install_localepurge_download() { + debrepo_add_packages "${DEBREPO_WORKDIR}" "localepurge" + update_apt_source_list "${ROOTFSDIR}" "base-apt" + sudo -E chroot '${ROOTFSDIR}' \ /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only localepurge } diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass index 2d3dda4f..a74820b8 100644 --- a/meta/classes/image-tools-extension.bbclass +++ b/meta/classes/image-tools-extension.bbclass @@ -6,6 +6,7 @@ # This file extends the image.bbclass to supply tools for futher imager functions inherit buildchroot +inherit debrepo IMAGER_INSTALL ??= "" IMAGER_BUILD_DEPS ??= "" @@ -21,14 +22,22 @@ do_install_imager_deps() { fi distro="${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" + debrepo_workdir=${DEBREPO_TARGET_DIR} if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}" + if [ ${HOST_ARCH} != ${DISTRO_ARCH} ]; then + debrepo_workdir=${DEBREPO_HOST_DIR} + fi fi buildchroot_do_mounts E="${@ isar_export_proxies(d)}" deb_dl_dir_import ${BUILDCHROOT_DIR} ${distro} + + debrepo_add_packages "${debrepo_workdir}" "${IMAGER_INSTALL}" + update_apt_source_list "${BUILDCHROOT_DIR}" "base-apt" + sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \ apt-get update \ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass index 92d665db..a3aec04c 100644 --- a/meta/classes/rootfs.bbclass +++ b/meta/classes/rootfs.bbclass @@ -2,6 +2,7 @@ # Copyright (c) Siemens AG, 2020 inherit deb-dl-dir +inherit debrepo ROOTFS_ARCH ?= "${DISTRO_ARCH}" ROOTFS_DISTRO ?= "${DISTRO}" @@ -143,6 +144,9 @@ rootfs_install_pkgs_download[weight] = "600" rootfs_install_pkgs_download[isar-apt-lock] = "release-after" rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK_AND_SUDO}" rootfs_install_pkgs_download() { + debrepo_add_packages "${DEBREPO_WORKDIR}" "${ROOTFS_PACKAGES}" + update_apt_source_list "${ROOTFSDIR}" "base-apt" + sudo -E chroot '${ROOTFSDIR}' \ /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES} } -- 2.20.1