[RFC 1/2] rootfs: download packages with host apt

[RFC 1/2] rootfs: download packages with host apt

['Felix Moessbauer' via isar-users]

The dependency resolving and download are architecture unspecific, hence
they do not need to be emulated. By not emulating them, we follow the
approach of mmdebstrap and significantly speedup this operation.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes-recipe/rootfs.bbclass | 40 ++++++++++++++----------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index e17f711f..6af29b40 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -337,10 +337,12 @@ rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadPro
 rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
 rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK}"
 rootfs_install_pkgs_download() {
-    # download packages using apt in a non-privileged namespace
-    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
-               ${ROOTFSDIR} \
-               -- /usr/bin/apt-get ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 --download-only ${ROOTFS_PACKAGES}
+    # download packages using the host apt in a non-privileged namespace.
+    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
+               -- /usr/bin/apt-get -o APT::Architecture=${ROOTFS_ARCH} \
+                                   -o Dir="${ROOTFSDIR}" \
+                                   ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 \
+                                   --download-only ${ROOTFS_PACKAGES}
 }
 
 ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
@@ -356,23 +358,19 @@ ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_isar_download"
 rootfs_install_pkgs_isar_download[weight] = "50"
 rootfs_install_pkgs_isar_download[isar-apt-lock] = "acquire-before release-after"
 rootfs_install_pkgs_isar_download() {
-    mkdir -p "${WORKDIR}/dpkg"
-
-    # Use our own dpkg lock files rather than those in the rootfs since we are not root
-    # (this is safe as there are no concurrent apt/dpkg operations for that rootfs)
-    touch "${WORKDIR}/dpkg/lock" "${WORKDIR}/dpkg/lock-frontend"
-
-    # Command apt-get install do not cache packages from local repos
-    # We can obtain non cached package URIs by recalling install command here
-    # No need in export those files to dl_dir, so we can run it right after
-    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
-               --bind "${WORKDIR}/dpkg/lock" /var/lib/dpkg/lock \
-               --bind "${WORKDIR}/dpkg/lock-frontend" /var/lib/dpkg/lock-frontend \
-               --chdir "/var/cache/apt/archives" \
-               ${ROOTFSDIR} \
-               -- /usr/bin/sh -c "apt-get ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
-                                  sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
-                                  xargs -r apt-get download"
+    # apt-get install does not cache packages coming from local repos
+    # (isar-apt). Obtain their URIs by recalling the install command and
+    # download them using the host apt, same as rootfs_install_pkgs_download.
+    # No need to export those files to dl_dir, so we can run it right after.
+    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
+               --chdir "${ROOTFSDIR}/var/cache/apt/archives" \
+               -- /usr/bin/sh -c "apt-get -o APT::Architecture=${ROOTFS_ARCH} \
+                                           -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
+                                           ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
+                                   sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
+                                   xargs -r apt-get -o APT::Architecture=${ROOTFS_ARCH} \
+                                                    -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
+                                                   download"
 }
 
 ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720105228.395364-1-felix.moessbauer%40siemens.com.

[RFC 2/2] rootfs: use host apt when installing in cross mode

['Felix Moessbauer' via isar-users]

Currently the apt inside the chroot is used to perform the extraction
and installation. This is really slow on non native architectures (esp.
on larger packages), as the whole operation is emulated.

We improve this by using the host apt for the dependency resolution,
unpack and configure operation. The dpkg chroots into the rootfs to run
the target maintainer scripts (emulated via qemu in cross builds),
mirroring how mmdebstrap installs. As we already have all packages in
the apt cache (since e09589d28), we don't need to mount the isar apt or
base apt - which would not work as the host apt would lookup the
absolute apt paths on the host (i.e. read /isar-apt from the host
instead of the chroot).

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/classes-recipe/rootfs.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
index 6af29b40..95210f4c 100644
--- a/meta/classes-recipe/rootfs.bbclass
+++ b/meta/classes-recipe/rootfs.bbclass
@@ -390,9 +390,15 @@ rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}"
 rootfs_install_pkgs_install() {
     run_privileged_heredoc <<'EOF'
     set -e
+    # Unpack and configure packages using the host apt. Dependency resolution
+    # and the apt frontend run natively (-o Dir points at the rootfs), while
+    # dpkg chroots into the rootfs to run the target maintainer scripts
+    # (emulated via qemu in cross builds), mirroring how mmdebstrap installs.
     ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), d.getVar('ROOTFS_MOUNTS')) if d.getVar('ISAR_CHROOT_MODE') == 'unshare' else ''}
-    chroot "${ROOTFSDIR}" \
-        /usr/bin/apt-get ${ROOTFS_APT_ARGS} --no-download ${ROOTFS_PACKAGES}
+    /usr/bin/apt-get -o APT::Architecture=${ROOTFS_ARCH} \
+        -o Dir="${ROOTFSDIR}" \
+        -o DPkg::Chroot-Directory="${ROOTFSDIR}" \
+        ${ROOTFS_APT_ARGS} --no-download ${ROOTFS_PACKAGES}
 EOF
 }
 
-- 
2.53.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20260720105228.395364-2-felix.moessbauer%40siemens.com.

Re: [RFC 2/2] rootfs: use host apt when installing in cross mode

[Anton Mikanovich]

20.07.2026 13:52, 'Felix Moessbauer' via isar-users wrote:
> Currently the apt inside the chroot is used to perform the extraction
> and installation. This is really slow on non native architectures (esp.
> on larger packages), as the whole operation is emulated.
>
> We improve this by using the host apt for the dependency resolution,
> unpack and configure operation. The dpkg chroots into the rootfs to run
> the target maintainer scripts (emulated via qemu in cross builds),
> mirroring how mmdebstrap installs. As we already have all packages in
> the apt cache (since e09589d28), we don't need to mount the isar apt or
> base apt - which would not work as the host apt would lookup the
> absolute apt paths on the host (i.e. read /isar-apt from the host
> instead of the chroot).
>
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>   meta/classes-recipe/rootfs.bbclass | 10 ++++++++--
>   1 file changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
> index 6af29b40..95210f4c 100644
> --- a/meta/classes-recipe/rootfs.bbclass
> +++ b/meta/classes-recipe/rootfs.bbclass
> @@ -390,9 +390,15 @@ rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}"
>   rootfs_install_pkgs_install() {
>       run_privileged_heredoc <<'EOF'
>       set -e
> +    # Unpack and configure packages using the host apt. Dependency resolution
> +    # and the apt frontend run natively (-o Dir points at the rootfs), while
> +    # dpkg chroots into the rootfs to run the target maintainer scripts
> +    # (emulated via qemu in cross builds), mirroring how mmdebstrap installs.
>       ${@insert_isar_mounts(d, d.getVar('ROOTFSDIR'), d.getVar('ROOTFS_MOUNTS')) if d.getVar('ISAR_CHROOT_MODE') == 'unshare' else ''}
> -    chroot "${ROOTFSDIR}" \
> -        /usr/bin/apt-get ${ROOTFS_APT_ARGS} --no-download ${ROOTFS_PACKAGES}
> +    /usr/bin/apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> +        -o Dir="${ROOTFSDIR}" \
> +        -o DPkg::Chroot-Directory="${ROOTFSDIR}" \
> +        ${ROOTFS_APT_ARGS} --no-download ${ROOTFS_PACKAGES}
I think we can drop full binary path here and just let the host OS use its
PATH value.
>   EOF
>   }
>   

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/9145ccb1-1679-497f-a467-e7dad05e964a%40ilbers.de.

Re: [RFC 1/2] rootfs: download packages with host apt

[Anton Mikanovich]

Hello Felix,

20.07.2026 13:52, 'Felix Moessbauer' via isar-users wrote:
> The dependency resolving and download are architecture unspecific, hence
> they do not need to be emulated. By not emulating them, we follow the
> approach of mmdebstrap and significantly speedup this operation.
>
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>   meta/classes-recipe/rootfs.bbclass | 40 ++++++++++++++----------------
>   1 file changed, 19 insertions(+), 21 deletions(-)
>
> diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
> index e17f711f..6af29b40 100644
> --- a/meta/classes-recipe/rootfs.bbclass
> +++ b/meta/classes-recipe/rootfs.bbclass
> @@ -337,10 +337,12 @@ rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadPro
>   rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
>   rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK}"
>   rootfs_install_pkgs_download() {
> -    # download packages using apt in a non-privileged namespace
> -    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
> -               ${ROOTFSDIR} \
> -               -- /usr/bin/apt-get ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 --download-only ${ROOTFS_PACKAGES}
> +    # download packages using the host apt in a non-privileged namespace.
> +    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
> +               -- /usr/bin/apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> +                                   -o Dir="${ROOTFSDIR}" \
> +                                   ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 \
> +                                   --download-only ${ROOTFS_PACKAGES}
Why do we need rootfs_cmd API usage then?
Doesn't it actually still use apt-get from ROOTFSDIR?
>   }
>   
>   ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
> @@ -356,23 +358,19 @@ ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_isar_download"
>   rootfs_install_pkgs_isar_download[weight] = "50"
>   rootfs_install_pkgs_isar_download[isar-apt-lock] = "acquire-before release-after"
>   rootfs_install_pkgs_isar_download() {
> -    mkdir -p "${WORKDIR}/dpkg"
> -
> -    # Use our own dpkg lock files rather than those in the rootfs since we are not root
> -    # (this is safe as there are no concurrent apt/dpkg operations for that rootfs)
> -    touch "${WORKDIR}/dpkg/lock" "${WORKDIR}/dpkg/lock-frontend"
> -
> -    # Command apt-get install do not cache packages from local repos
> -    # We can obtain non cached package URIs by recalling install command here
> -    # No need in export those files to dl_dir, so we can run it right after
> -    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
> -               --bind "${WORKDIR}/dpkg/lock" /var/lib/dpkg/lock \
> -               --bind "${WORKDIR}/dpkg/lock-frontend" /var/lib/dpkg/lock-frontend \
> -               --chdir "/var/cache/apt/archives" \
> -               ${ROOTFSDIR} \
> -               -- /usr/bin/sh -c "apt-get ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
> -                                  sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
> -                                  xargs -r apt-get download"
> +    # apt-get install does not cache packages coming from local repos
> +    # (isar-apt). Obtain their URIs by recalling the install command and
> +    # download them using the host apt, same as rootfs_install_pkgs_download.
> +    # No need to export those files to dl_dir, so we can run it right after.
> +    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
> +               --chdir "${ROOTFSDIR}/var/cache/apt/archives" \
> +               -- /usr/bin/sh -c "apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> +                                           -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
> +                                           ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
> +                                   sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
> +                                   xargs -r apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> +                                                    -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
> +                                                   download"
Besides of the same question, this task rebuild will probably also helps 
with
ignoring missing deb file arch in the existing implementation, which can
possibly fail if we try to build custom package for both host and target
architectures.
>   }
>   
>   ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/9a53f70e-f23a-4a49-885c-da3177efed11%40ilbers.de.

Re: [RFC 1/2] rootfs: download packages with host apt

['MOESSBAUER, Felix' via isar-users]

On Tue, 2026-07-21 at 16:56 +0300, Anton Mikanovich wrote:
> Hello Felix,
> 
> 20.07.2026 13:52, 'Felix Moessbauer' via isar-users wrote:
> > The dependency resolving and download are architecture unspecific, hence
> > they do not need to be emulated. By not emulating them, we follow the
> > approach of mmdebstrap and significantly speedup this operation.
> > 
> > Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> > ---
> >    meta/classes-recipe/rootfs.bbclass | 40 ++++++++++++++----------------
> >    1 file changed, 19 insertions(+), 21 deletions(-)
> > 
> > diff --git a/meta/classes-recipe/rootfs.bbclass b/meta/classes-recipe/rootfs.bbclass
> > index e17f711f..6af29b40 100644
> > --- a/meta/classes-recipe/rootfs.bbclass
> > +++ b/meta/classes-recipe/rootfs.bbclass
> > @@ -337,10 +337,12 @@ rootfs_install_pkgs_download[progress] = "custom:rootfs_progress.PkgsDownloadPro
> >    rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
> >    rootfs_install_pkgs_download[network] = "${TASK_USE_NETWORK}"
> >    rootfs_install_pkgs_download() {
> > -    # download packages using apt in a non-privileged namespace
> > -    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
> > -               ${ROOTFSDIR} \
> > -               -- /usr/bin/apt-get ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 --download-only ${ROOTFS_PACKAGES}
> > +    # download packages using the host apt in a non-privileged namespace.
> > +    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
> > +               -- /usr/bin/apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> > +                                   -o Dir="${ROOTFSDIR}" \
> > +                                   ${ROOTFS_APT_ARGS} -o Debug::NoLocking=1 \
> > +                                   --download-only ${ROOTFS_PACKAGES}
> Why do we need rootfs_cmd API usage then?
> Doesn't it actually still use apt-get from ROOTFSDIR?

We at least need it to bind-mount /isar-apt and /dev, ... on unshare.
But you're right, the /bin, /etc, /lib, ... mounts look scary.

Maybe Cedric can help to get the implementation right.
I just sent this as an RFC, as I want to get feedback about the general
approach to use the host apt for more things (we are anyways already
doing this via mmdebstrap).

> >    }
> >    
> >    ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
> > @@ -356,23 +358,19 @@ ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_isar_download"
> >    rootfs_install_pkgs_isar_download[weight] = "50"
> >    rootfs_install_pkgs_isar_download[isar-apt-lock] = "acquire-before release-after"
> >    rootfs_install_pkgs_isar_download() {
> > -    mkdir -p "${WORKDIR}/dpkg"
> > -
> > -    # Use our own dpkg lock files rather than those in the rootfs since we are not root
> > -    # (this is safe as there are no concurrent apt/dpkg operations for that rootfs)
> > -    touch "${WORKDIR}/dpkg/lock" "${WORKDIR}/dpkg/lock-frontend"
> > -
> > -    # Command apt-get install do not cache packages from local repos
> > -    # We can obtain non cached package URIs by recalling install command here
> > -    # No need in export those files to dl_dir, so we can run it right after
> > -    rootfs_cmd --bind "${ROOTFSDIR}/var/cache/apt/archives" /var/cache/apt/archives \
> > -               --bind "${WORKDIR}/dpkg/lock" /var/lib/dpkg/lock \
> > -               --bind "${WORKDIR}/dpkg/lock-frontend" /var/lib/dpkg/lock-frontend \
> > -               --chdir "/var/cache/apt/archives" \
> > -               ${ROOTFSDIR} \
> > -               -- /usr/bin/sh -c "apt-get ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
> > -                                  sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
> > -                                  xargs -r apt-get download"
> > +    # apt-get install does not cache packages coming from local repos
> > +    # (isar-apt). Obtain their URIs by recalling the install command and
> > +    # download them using the host apt, same as rootfs_install_pkgs_download.
> > +    # No need to export those files to dl_dir, so we can run it right after.
> > +    rootfs_cmd --bind "${ROOTFSDIR}" "${ROOTFSDIR}" \
> > +               --chdir "${ROOTFSDIR}/var/cache/apt/archives" \
> > +               -- /usr/bin/sh -c "apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> > +                                           -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
> > +                                           ${ROOTFS_APT_ARGS} --print-uris ${ROOTFS_PACKAGES} | \
> > +                                   sed -n \"s|^.*/\\(.*\\)_[^_]*_[^_]*\\.deb'.*|\\1|p\" | \
> > +                                   xargs -r apt-get -o APT::Architecture=${ROOTFS_ARCH} \
> > +                                                    -o \"Dir=${ROOTFSDIR}\" -o Debug::NoLocking=1 \
> > +                                                   download"
> Besides of the same question, this task rebuild will probably also helps 
> with
> ignoring missing deb file arch in the existing implementation, which can
> possibly fail if we try to build custom package for both host and target
> architectures.

Probably, because we are explicit. But I don't know of any bugs we
currently have regarding this.

Felix

> >    }
> >    
> >    ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/5da1e31182c28648a188de3146e71131e7878b6d.camel%40siemens.com.