[PATCH v4] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot

[PATCH v4] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot

[Jan Kiszka]

From: Jan Kiszka <jan.kiszka@siemens.com>

During the migration to sbuild, support for using self-built packages in
the build environment was lost if those were already part of the
sbuild-chroot. This restores it by adding --apt-distupgrade to the
sbuild call. But that is not enough because sbuild will only upgrade
packages from already configured sources, not those specified via
--extra-repository. We therefore have to switch back to configuring
isar-apt during sbuild-chroot creation.

As rootfs_configure_isar_apt configures the isar repo under
/isar-apt, we bind-mount the one in /home/builder/${PN} to that folder.
Another difference is that we now need to run apt-get update explicitly,
but only for isar-apt.

And because the sbuild-chroots are now left behind with isar-apt
configured, the configuration in imager_run can be removed (credits to
Srinuvasan Arjunan).

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---

Changes in v4:
 - pick up cleanup of image-tool-extensions by Srinu

 meta/classes/dpkg-base.bbclass                | 27 ++++++++++---------
 meta/classes/dpkg.bbclass                     |  5 ++--
 meta/classes/image-tools-extension.bbclass    | 13 ---------
 .../sbuild-chroot/sbuild-chroot.inc           |  2 --
 4 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 7b054d3f..52baf169 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -19,8 +19,6 @@ DEPENDS:append:riscv64 = "${@' crossbuild-essential-riscv64' if d.getVar('ISAR_C
 DEB_BUILD_PROFILES ?= ""
 DEB_BUILD_OPTIONS ?= ""
 
-ISAR_APT_REPO ?= "deb [trusted=yes] file:///home/builder/${PN}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} ${DEBDISTRONAME} main"
-
 python do_adjust_git() {
     import subprocess
 
@@ -221,8 +219,17 @@ def isar_export_build_settings(d):
     os.environ['DEB_BUILD_OPTIONS']  = isar_deb_build_options(d)
     os.environ['DEB_BUILD_PROFILES'] = isar_deb_build_profiles(d)
 
+dpkg_schroot_create_configs() {
+    schroot_create_configs
+    sudo -s <<'EOSUDO'
+        sbuild_fstab="${SBUILD_CONF_DIR}/fstab"
+        fstab_isarapt="${WORKDIR}/isar-apt/${DISTRO}-${DISTRO_ARCH}/apt/${DISTRO} /isar-apt none rw,bind 0 0"
+        grep -qxF "${fstab_isarapt}" ${sbuild_fstab} || echo "${fstab_isarapt}" >> ${sbuild_fstab}
+EOSUDO
+}
+
 python do_dpkg_build() {
-    bb.build.exec_func('schroot_create_configs', d)
+    bb.build.exec_func('dpkg_schroot_create_configs', d)
     try:
         bb.build.exec_func("dpkg_runbuild", d)
     finally:
@@ -288,7 +295,7 @@ do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
 do_deploy_deb[dirs] = "${S}"
 
 python do_devshell() {
-    bb.build.exec_func('schroot_create_configs', d)
+    bb.build.exec_func('dpkg_schroot_create_configs', d)
 
     isar_export_proxies(d)
     isar_export_ccache(d)
@@ -297,11 +304,9 @@ python do_devshell() {
         bb.build.exec_func('schroot_configure_ccache', d)
 
     schroot = d.getVar('SBUILD_CHROOT')
-    isar_apt = d.getVar('ISAR_APT_REPO')
     pkg_arch = d.getVar('PACKAGE_ARCH')
     build_arch = d.getVar('BUILD_ARCH')
     pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
-    debdistroname = d.getVar('DEBDISTRONAME')
 
     install_deps = ":" if d.getVar('BB_CURRENTTASK') == "devshell_nodeps" else f"mk-build-deps -i \
         --host-arch {pkg_arch} --build-arch {build_arch}  \
@@ -310,15 +315,13 @@ python do_devshell() {
 
     termcmd = "schroot -d / -c {0} -u root -- sh -c ' \
         cd {1}; \
-        echo {2} > /etc/apt/sources.list.d/isar_apt.list; \
-        echo \"Package: *\nPin: release n={3}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt; \
-        echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt; \
-        apt-get -y -q update; \
-        {4}; \
+        apt-get -y -q update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"; \
+        apt-get -y upgrade; \
+        {2}; \
         export PATH=$PATH_PREPEND:$PATH; \
         $SHELL -i \
     '"
-    oe_terminal(termcmd.format(schroot, pp_pps, isar_apt, debdistroname, install_deps), "Isar devshell", d)
+    oe_terminal(termcmd.format(schroot, pp_pps, install_deps), "Isar devshell", d)
 
     bb.build.exec_func('schroot_delete_configs', d)
 }
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 1d684e98..0cdb5792 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -110,16 +110,17 @@ dpkg_runbuild() {
     DEB_SOURCE_NAME=$(dpkg-parsechangelog --show-field Source --file ${WORKDIR}/${PPS}/debian/changelog)
     DSC_FILE=$(find ${WORKDIR} -name "${DEB_SOURCE_NAME}*.dsc" -maxdepth 1 -print)
 
-    sbuild -A -n -c ${SBUILD_CHROOT} --extra-repository="${ISAR_APT_REPO}" \
+    sbuild -A -n -c ${SBUILD_CHROOT} \
         --host=${PACKAGE_ARCH} --build=${BUILD_ARCH} ${profiles} \
         --no-run-lintian --no-run-piuparts --no-run-autopkgtest --resolve-alternatives \
         --bd-uninstallable-explainer=apt \
-        --no-apt-update \
+        --no-apt-update --apt-distupgrade \
         --chroot-setup-commands="echo \"Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000\" > /etc/apt/preferences.d/isar-apt" \
         --chroot-setup-commands="echo \"APT::Get::allow-downgrades 1;\" > /etc/apt/apt.conf.d/50isar-apt" \
         --chroot-setup-commands="rm -f /var/log/dpkg.log" \
         --chroot-setup-commands="mkdir -p ${deb_dir}" \
         --chroot-setup-commands="ln -sf ${ext_deb_dir}/*.deb -t ${deb_dir}/" \
+        --chroot-setup-commands="apt-get update -o Dir::Etc::SourceList=\"sources.list.d/isar-apt.list\" -o Dir::Etc::SourceParts=\"-\" -o APT::Get::List-Cleanup=\"0\"" \
         --finished-build-commands="rm -f ${deb_dir}/sbuild-build-depends-main-dummy_*.deb" \
         --finished-build-commands="[ -z "$(find ${deb_dir} -maxdepth 1 -name '*.deb' -print -quit)" ] || cp ${CP_FLAGS} ${deb_dir}/*.deb -t ${ext_deb_dir}/" \
         --finished-build-commands="cp /var/log/dpkg.log ${ext_root}/dpkg_partial.log" \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 58799fcf..46bdf78b 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -44,19 +44,6 @@ imager_run() {
             distro="${HOST_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
         fi
 
-        # prepare isar-apt
-        schroot -r -c ${session_id} -d / -u root -- sh -c " \
-            mkdir -p '/etc/apt/sources.list.d'
-            echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
-                '/etc/apt/sources.list.d/isar-apt.list'
-
-            mkdir -p '/etc/apt/preferences.d'
-            cat << EOF > '/etc/apt/preferences.d/isar-apt'
-Package: *
-Pin: release n=${DEBDISTRONAME}
-Pin-Priority: 1000
-EOF"
-
         E="${@ isar_export_proxies(d)}"
         deb_dl_dir_import ${schroot_dir} ${distro}
         schroot -r -c ${session_id} -d / -u root -- sh -c " \
diff --git a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
index 39ced80b..2f07de82 100644
--- a/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
+++ b/meta/recipes-devtools/sbuild-chroot/sbuild-chroot.inc
@@ -56,8 +56,6 @@ SBUILD_CHROOT_DIR = "${WORKDIR}/rootfs"
 ROOTFSDIR = "${SBUILD_CHROOT_DIR}"
 ROOTFS_PACKAGES = "${SBUILD_CHROOT_PREINSTALL}"
 
-# We don't need /etc/apt/sources.list.d/isar-apt.list' while it's handled by sbuild
-ROOTFS_CONFIGURE_COMMAND:remove = "rootfs_configure_isar_apt"
 ROOTFS_POSTPROCESS_COMMAND:remove = "rootfs_cleanup_isar_apt"
 
 # Do not cleanup base-apt
-- 
2.35.3

Re: [PATCH v4] dpkg: Restore support for replacing pre-installed packages in sbuild-chroot

[Uladzimir Bely]

[-- Attachment #1: Type: text/plain, Size: 1774 bytes --]

On Mon, 2024-01-08 at 07:50 +0100, 'Jan Kiszka' via isar-users wrote:
> From: Jan Kiszka <jan.kiszka@siemens.com>
> 
> During the migration to sbuild, support for using self-built packages
> in
> the build environment was lost if those were already part of the
> sbuild-chroot. This restores it by adding --apt-distupgrade to the
> sbuild call. But that is not enough because sbuild will only upgrade
> packages from already configured sources, not those specified via
> --extra-repository. We therefore have to switch back to configuring
> isar-apt during sbuild-chroot creation.
> 
> As rootfs_configure_isar_apt configures the isar repo under
> /isar-apt, we bind-mount the one in /home/builder/${PN} to that
> folder.
> Another difference is that we now need to run apt-get update
> explicitly,
> but only for isar-apt.
> 
> And because the sbuild-chroots are now left behind with isar-apt
> configured, the configuration in imager_run can be removed (credits
> to
> Srinuvasan Arjunan).
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 

The patch for now doesn't pass full CI and fails with similar errors in
several tests (repro and container related). I've attached some log
files. In addition to the logs, I have to say that isar-apt copy in
workdir (e.g. "hello/1.0-r0/isar-apt/...") does not contain required
file in the pool (e.g. "/isar-apt/pool/main/h/hello/hello_2.10-
3+isar.dsc").


I'm still debugging it for the reason of the build issue, but probably
the logs attached could be useful for understanding what is wrong.

For now, the simplest way to reproduce the issue is running CI limited
to repro_unsigned test (target list can also be reduced up to
mc:qemuamd64-bullseye:isar-image-base only).

[-- Attachment #2: build.tar.bz2 --]
[-- Type: application/x-bzip2-compressed-tar, Size: 22601 bytes --]