[PATCH 0/4] Bootstrap from local isar-apt repo

[PATCH 0/4] Bootstrap from local isar-apt repo

[Andreas Naumann]

We have a use case where the bootstrapping takes place using the
contents of the internal isar-apt repo only and not depend on an
external distro repo. (This does not exclude depending on the
underlying fileserver).

In order to achieve that, an number of dpkg-prebuilt packages
are created and deployed to isar-apt before bootstrapping. (out of
scope for this patchset).

The patchset here is to introduce an option which allows for isar-apt
to be used instead of a remote distro source repo.

Notes:
ISAR_USE_CACHED_BASE_REPO was considered but unfortunately is not
exactly what's needed.


Andreas Naumann (4):
  mmdebstrap: Make use of defined variables
  mmdebstrap: Include isar-apt in bootstrapping sources
  mmdebstrap: Allow for DISTRO_SOURCE to be empty
  mmdebstrap: lock isar-apt while bootstrapping

 meta/classes/bootstrap.bbclass                |  2 +
 .../isar-mmdebstrap/isar-mmdebstrap.inc       | 37 +++++++++++++------
 2 files changed, 28 insertions(+), 11 deletions(-)

-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250812130249.709960-1-anaumann%40emlix.com.

[PATCH 1/4] mmdebstrap: Make use of defined variables

[Andreas Naumann]

Signed-off-by: Andreas Naumann <anaumann@emlix.com>
---
 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 931f6f138e..42d99756a1 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -106,12 +106,12 @@ do_bootstrap() {
         if [ -z "${BASE_REPO_KEY}" ]; then
             line="[trusted=yes] ${line}"
         fi
-        echo "deb ${line}" >  "${WORKDIR}/sources.list.d/base-apt.list"
+        echo "deb ${line}" >  "$bootstrap_list"
         line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
         if [ -z "${BASE_REPO_KEY}" ]; then
             line="[trusted=yes] ${line}"
         fi
-        echo "deb-src ${line}" >>  "${WORKDIR}/sources.list.d/base-apt.list"
+        echo "deb-src ${line}" >>  "$bootstrap_list"
 
         # no need to sync /var/cache/apt/archives if base-apt used
         syncin='echo skip sync-in'
@@ -137,8 +137,7 @@ do_bootstrap() {
         deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
 
         bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
-        install -v -m644 "${APTSRCS}" \
-                         "${WORKDIR}/sources.list.d/bootstrap.list"
+        install -v -m644 "${APTSRCS}" "$bootstrap_list"
 
         syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
                       "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250812130249.709960-2-anaumann%40emlix.com.

[PATCH 2/4] mmdebstrap: Include isar-apt in bootstrapping sources

[Andreas Naumann]

To support using custom variants of packages in bootstrap, include isar-apt
as possible distro source.
In order to avoid wasting time on copying, reuse the bind-mount technique
from the base-apt implementation.

Signed-off-by: Andreas Naumann <anaumann@emlix.com>
---
 .../isar-mmdebstrap/isar-mmdebstrap.inc       | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index 42d99756a1..b03ae7d772 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -136,8 +136,11 @@ do_bootstrap() {
     else
         deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
 
+        isar_apt_tmp="$(mktemp -d /tmp/isar-aptXXXXXXXXXX)"
         bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
-        install -v -m644 "${APTSRCS}" "$bootstrap_list"
+        line="copy://$isar_apt_tmp/${DISTRO} ${DEBDISTRONAME} main"
+        echo "deb [trusted=yes] ${line}" >  "$bootstrap_list"
+        cat "${APTSRCS}" >> "$bootstrap_list"
 
         syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
                       "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
@@ -145,17 +148,23 @@ do_bootstrap() {
         syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
                        "$1/var/cache/apt/archives/"*.deb \
                        "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
-        extra_setup="$syncin"
+        extra_setup="mount --bind '${REPO_ISAR_DIR}' $isar_apt_tmp"
+        extra_setup="$extra_setup && $syncin"
         extra_extract="$syncout"
+        extra_essential="mkdir -p \$1/$isar_apt_tmp && \
+                         echo \$1 > ${WORKDIR}/mmtmpdir && \
+                         mount -o bind,private '${REPO_ISAR_DIR}' \$1/$isar_apt_tmp"
         # prefetch apt debs because mmdebstrap will clean them on next stage
-        extra_essential='apt-get install apt -y -d \
+        extra_apt='apt-get install apt -y -d \
                                  -o Dir::State="$1/var/lib/apt" \
                                  -o Dir::Etc="$1/etc/apt" \
                                  -o Dir::Cache="$1/var/cache/apt" \
                                  -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}" \
                                  ${@get_apt_opts(d, '-o')}'
-        extra_essential="$extra_essential && $syncout"
-        extra_customize="$syncout"
+        extra_essential="$extra_essential && $extra_apt && $syncout"
+        extra_customize="umount \$1/$isar_apt_tmp && \
+                         umount $isar_apt_tmp && rm -rf --one-file-system $isar_apt_tmp"
+        extra_customize="$extra_customize && $syncout"
     fi
 
     if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
@@ -178,7 +187,10 @@ do_bootstrap() {
           [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
           [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
                                  && sudo umount $base_apt_tmp \
-                                 && rm -rf --one-file-system $base_apt_tmp' EXIT
+                                 && rm -rf --one-file-system $base_apt_tmp; \
+          [ -n "$isar_apt_tmp" ] && mountpoint -q $isar_apt_tmp \
+                                 && sudo umount $isar_apt_tmp \
+                                 && rm -rf --one-file-system $isar_apt_tmp' EXIT
 
     sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \
                    $arch_param \
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250812130249.709960-3-anaumann%40emlix.com.

[PATCH 3/4] mmdebstrap: Allow for DISTRO_SOURCE to be empty

[Andreas Naumann]

For isar-apt to be actually used, mmdebstrap needs to be supplied
with the correct suite name, "isar" as default.

Introduce a flag which allows switching to use the suite of the
internal isar-apt instead of DISTRO_SOURCE

Signed-off-by: Andreas Naumann <anaumann@emlix.com>
---
 meta/classes/bootstrap.bbclass | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/meta/classes/bootstrap.bbclass b/meta/classes/bootstrap.bbclass
index 64702d5d54..ae1dec9318 100644
--- a/meta/classes/bootstrap.bbclass
+++ b/meta/classes/bootstrap.bbclass
@@ -179,6 +179,8 @@ def generate_distro_sources(d):
                     yield parsed
 
 def get_distro_primary_source_entry(d):
+    if bb.utils.to_boolean(d.getVar('ISAR_USE_ISAR_APT_TO_BOOTSTRAP')):
+        return ["", "${DEBDISTRONAME}", "main"]
     for source in generate_distro_sources(d):
         if source[0] == "deb":
             return source[2:]
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250812130249.709960-4-anaumann%40emlix.com.

[PATCH 4/4] mmdebstrap: lock isar-apt while bootstrapping

[Andreas Naumann]

isar-apt must not change while we bootstrap from it. Otherwise, e.g.
if a dpkg recipe deploys a deb-file into it, the Releases or Packages
file may not match the hash which causes apt to stop with an error.

In order to use the bitbake lockfile flag on the entire task, which
is preferred to a flock-wrap around the mmdebstrap command, move code
with conflicting locking to a new finalize task.

Signed-off-by: Andreas Naumann <anaumann@emlix.com>
---
 meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
index b03ae7d772..885963fcc4 100644
--- a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
+++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
@@ -86,6 +86,7 @@ do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
 DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
 
 do_bootstrap[vardeps] += "DISTRO_BOOTSTRAP_BASE_PACKAGES"
+do_bootstrap[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
 do_bootstrap() {
     if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
         if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -217,7 +218,10 @@ do_bootstrap() {
                    "${@get_distro_suite(d)}" \
                    "${WORKDIR}/rootfs.tar.zst" \
                    "$bootstrap_list"
+}
+addtask bootstrap before do_build after do_generate_keyrings
 
+do_bootstrap_finalize() {
     # Finalize bootstrap by setting the link in deploy
     sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
 
@@ -226,9 +230,9 @@ do_bootstrap() {
         sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
     fi
 }
-addtask bootstrap before do_build after do_generate_keyrings
+addtask do_bootstrap_finalize after do_bootstrap before do_build
 
-SSTATETASKS += "do_bootstrap"
+SSTATETASKS += "do_bootstrap do_bootstrap_finalize"
 SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
 SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
 
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/20250812130249.709960-5-anaumann%40emlix.com.