public inbox for isar-users@googlegroups.com
 help / color / mirror / Atom feed
* [PATCH v5] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot
@ 2024-01-16  8:22 Jan Kiszka
  2024-01-18  6:28 ` Uladzimir Bely
  0 siblings, 1 reply; 7+ messages in thread
From: Jan Kiszka @ 2024-01-16  8:22 UTC (permalink / raw)
  To: isar-users; +Cc: A, Srinuvasan, Uladzimir Bely

From: Jan Kiszka <jan.kiszka@siemens.com>

During the migration to sbuild, support for using self-built packages in
the build environment was lost if those were already part of the
sbuild-chroot. This restores it by adding --apt-distupgrade to the
sbuild call. But that is not enough because sbuild will only upgrade
packages from already configured sources, not those specified via
--extra-repository. We therefore have to switch back to configuring
isar-apt during sbuild-chroot creation.

As rootfs_configure_isar_apt configures the isar repo under
/isar-apt, we bind-mount the one in /home/builder/${PN} to that folder.
Another difference is that we now need to run apt-get update explicitly,
but only for isar-apt.

And because the sbuild-chroots are now left behind with isar-apt
configured, the configuration in imager_run can be removed (credits to
Srinuvasan Arjunan).

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Changes in v5:
 - also use dpkg_schroot_create_configs for apt_fetch and apt_unpack
   (while I wasn't able to reproduce the issue, I think this should
   resolve the CI problems)

 meta/classes/dpkg-base.bbclass                | 66 ++++++++++---------
 meta/classes/dpkg.bbclass                     |  5 +-
 meta/classes/image-tools-extension.bbclass    | 13 ----
 .../sbuild-chroot/sbuild-chroot.inc           |  2 -
 4 files changed, 38 insertions(+), 48 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 7b054d3f..32494636 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C
 DEB_BUILD_PROFILES ?= ""
 DEB_BUILD_OPTIONS ?= ""
 
-ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main"
-
 python do_adjust_git() {
     import subprocess
 
@@ -105,9 +103,34 @@ python() {
         bb.build.addtask('cleanall_apt', 'do_cleanall', '', d)
 }
 
+dpkg_schroot_create_configs() {
+    schroot_create_configs
+    sudo -s <<'EOSUDO'
+        sbuild_fstab="${SBUILD_CONF_DIR}/fstab"
+        fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0"
+        grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab}
+EOSUDO
+}
+
+# If Isar recipes depend on each other, they typically need the package
+# deployed to isar-apt
+addtask local_isarapt
+do_local_isarapt[depends] += "isar-apt:do_cache_config"
+do_local_isarapt[deptask] = "do_deploy_deb"
+do_local_isarapt[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
+do_local_isarapt() {
+    # Make a local copy of isar-apt repo that is not affected by other parallel builds
+    rm -rf "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/*"
+    mkdir -p "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}"
+    cp -Rf "${REPO_ISAR_DIR}/${DISTRO}/dists" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
+    if [ -d "${REPO_ISAR_DIR}/${DISTRO}/pool" ]; then
+        cp -Rlf "${REPO_ISAR_DIR}/${DISTRO}/pool" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
+    fi
+}
+
 do_apt_fetch() {
     E="${@ isar_export_proxies(d)}"
-    schroot_create_configs
+    dpkg_schroot_create_configs
 
     schroot_cleanup() {
         schroot_delete_configs
@@ -122,7 +145,7 @@ do_apt_fetch() {
     schroot_delete_configs
 }
 
-addtask apt_fetch
+addtask apt_fetch after do_local_isarapt
 do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
 do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 
@@ -131,7 +154,7 @@ do_apt_fetch[depends] += "${SCHROOT_DEP}"
 
 do_apt_unpack() {
     rm -rf ${S}
-    schroot_create_configs
+    dpkg_schroot_create_configs
 
     schroot_cleanup() {
         schroot_delete_configs
@@ -184,21 +207,6 @@ do_prepare_build() {
 }
 
 addtask prepare_build after do_patch do_transform_template before do_dpkg_build
-# If Isar recipes depend on each other, they typically need the package
-# deployed to isar-apt
-do_local_isarapt[depends] += "isar-apt:do_cache_config"
-do_local_isarapt[deptask] = "do_deploy_deb"
-do_local_isarapt[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
-do_local_isarapt() {
-    # Make a local copy of isar-apt repo that is not affected by other parallel builds
-    rm -rf "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/*"
-    mkdir -p "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}"
-    cp -Rf "${REPO_ISAR_DIR}/${DISTRO}/dists" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
-    if [ -d "${REPO_ISAR_DIR}/${DISTRO}/pool" ]; then
-        cp -Rlf "${REPO_ISAR_DIR}/${DISTRO}/pool" "${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO}/"
-    fi
-}
-addtask local_isarapt before do_dpkg_build
 
 # Placeholder for actual dpkg_runbuild() implementation
 dpkg_runbuild() {
@@ -222,7 +230,7 @@ def isar_export_build_settings(d):
     os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d)
 
 python do_dpkg_build() {
-    bb.build.exec_func('schroot_create_configs', d)
+    bb.build.exec_func('dpkg_schroot_create_configs', d)
     try:
         bb.build.exec_func("dpkg_runbuild", d)
     finally:
@@ -230,7 +238,7 @@ python do_dpkg_build() {
 }
 do_dpkg_build[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 
-addtask dpkg_build
+addtask dpkg_build after do_local_isarapt
 
 SSTATETASKS += "do_dpkg_build"
 SSTATECREATEFUNCS += "dpkg_build_sstate_prepare"
@@ -288,7 +296,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
 do_deploy_deb[dirs] = "${S}"
 
 python do_devshell() {
-    bb.build.exec_func('schroot_create_configs', d)
+    bb.build.exec_func('dpkg_schroot_create_configs', d)
 
     isar_export_proxies(d)
     isar_export_ccache(d)
@@ -297,11 +305,9 @@ python do_devshell() {
         bb.build.exec_func('schroot_configure_ccache', d)
 
     schroot = d.getVar('SBUILD_CHROOT')
-    isar_apt = d.getVar('ISAR_APT_REPO')
     pkg_arch = d.getVar('PACKAGE_ARCH')
     build_arch = d.getVar('BUILD_ARCH')
     pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
-    debdistroname = d.getVar('DEBDISTRONAME')
 
     install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \
         --host-arch {pkg_arch} --build-arch {build_arch}  \
@@ -310,15 +316,13 @@ python do_devshell() {
 
     termcmd = "schroot -d / -c {0} -u root -- sh -c ' \
         cd {1}; \
-        echo {2} > /etc/apt/sources.list.d/isar_apt.list; \
-        echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \
-        echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \
-        apt-get -y -q update; \
-        {4}; \
+        apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \
+        apt-get -y upgrade; \
+        {2}; \
         export PATH=$PATH_PREPEND:$PATH; \
         $SHELL -i \
     '"
-    oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d)
+    oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d)
 
     bb.build.exec_func('schroot_delete_configs', d)
 }
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index c699a84d..d1666f78 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -110,16 +110,17 @@ dpkg_runbuild() {
     DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog)
     DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print)
 
-    sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \
+    sbuild -A -n -c ${SBUILD_CHROOT} \
         --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \
         --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \
         --bd-uninstallable-explainer=apt \
-        --no-apt-update \
+        --no-apt-update --apt-distupgrade \
         --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \
         --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
         --chroot-setup-commands="mkdir -p ${deb_dir}" \
         --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \
+        --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
         --finished-build-commands="[ -z \"\$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)\" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \
         --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 58799fcf..46bdf78b 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -44,19 +44,6 @@ imager_run() {
             distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
         fi
 
-        # prepare isar-apt
-        schroot -r -c ${session_id} -d / -u root -- sh -c " \
-            mkdir -p '/etc/apt/sources.list.d'
-            echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
-                '/etc/apt/sources.list.d/isar-apt.list'
-
-            mkdir -p '/etc/apt/preferences.d'
-            cat << EOF > '/etc/apt/preferences.d/isar-apt'
-Package: *
-Pin: release n=${DEBDISTRONAME}
-Pin-Priority: 1000
-EOF"
-
         E="${@ isar_export_proxies(d)}"
         deb_dl_dir_import ${schroot_dir} ${distro}
         schroot -r -c ${session_id} -d / -u root -- sh -c " \
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
index 39ced80b..2f07de82 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
@@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs"
 ROOTFSDIR = "${SBUILD_CHROOT_DIR}"
 ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}"
 
-# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild
-ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt"
 ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt"
 
 # Do not cleanup base-apt
-- 
2.35.3

^ permalink raw reply	[flat|nested] 7+ messages in thread
end of thread, other threads:[~2024-01-18 10:28 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-01-16  8:22 [PATCH v5] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot Jan Kiszka
2024-01-18  6:28 ` Uladzimir Bely
2024-01-18  6:34   ` Uladzimir Bely
2024-01-18  6:59     ` Jan Kiszka
2024-01-18  7:39       ` Uladzimir Bely
2024-01-18  7:54         ` Jan Kiszka
2024-01-18 10:28         ` Jan Kiszka

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox