* [PATCH v2 1/6] meta: Switch to mmdebstrap
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-20 4:50 ` MOESSBAUER, Felix
2023-05-18 9:47 ` [PATCH v2 2/6] isar-bootstrap: Move preparations to hooks Anton Mikanovich
` (5 subsequent siblings)
6 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Uladzimir Bely, Anton Mikanovich
From: Uladzimir Bely <ubely@ilbers.de>
Instead of debootstrap, use mmdebstrap alternative.
Internally, it uses apt and allows to debootstrap the distro from
multiple repositories.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta-isar/conf/distro/ubuntu-focal.conf | 4 --
.../isar-bootstrap/isar-bootstrap.inc | 57 ++++++++++---------
2 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
index 0cb6958d..be8c951c 100644
--- a/meta-isar/conf/distro/ubuntu-focal.conf
+++ b/meta-isar/conf/distro/ubuntu-focal.conf
@@ -17,10 +17,6 @@ BOOTSTRAP_KEY = "file://${LAYERDIR_isar}/conf/distro/ubuntu.public.key;sha256sum
DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
-
-# that is what debootstrap_1.0.118ubuntu1 does anyways
-DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
-
DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
DISTRO_KERNELS ?= " \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index b94ae0bd..f89c284e 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -15,7 +15,7 @@ PV = "1.0"
BOOTSTRAP_FOR_HOST ?= "0"
-DEBOOTSTRAP ?= "qemu-debootstrap"
+DEBOOTSTRAP = "mmdebstrap"
ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
@@ -23,12 +23,14 @@ APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
-DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
+DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
+BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
+BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
FILESEXTRAPATHS:append = ":${BBPATH}"
inherit deb-dl-dir
@@ -270,7 +272,7 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
@@ -286,27 +288,31 @@ do_bootstrap() {
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
fi
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z "${BASE_REPO_KEY}" ]; then
- debootstrap_args="$debootstrap_args --no-check-gpg"
- fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST debootstrap_args E
+ export BOOTSTRAP_FOR_HOST
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${ROOTFSDIR}"
+ mkdir -p "${ROOTFSDIR}"
+
+ if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
+ export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+ fi
+
+ arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+
+ sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
+ $arch_param \
+ --mode=unshare \
+ ${@get_distro_components_argument(d)} \
+ "${@get_distro_suite(d)}" \
+ "${WORKDIR}/rootfs.tar.zst" \
+ "${@get_distro_source(d)}"
sudo -E -s <<'EOSUDO'
set -e
- if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
- arch_param="--arch=${DISTRO_ARCH}"
- fi
- ${DEBOOTSTRAP} $debootstrap_args \
- $arch_param \
- ${@get_distro_components_argument(d)} \
- "${@get_distro_suite(d)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d)}" \
- ${DISTRO_DEBOOTSTRAP_SCRIPT}
+
+ tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
# Install apt config
mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
@@ -409,19 +415,16 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- lopts="--one-file-system --exclude=var/cache/apt/archives"
- sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
- sudo chown $(id -u):$(id -g) bootstrap.tar
+ sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
- # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
- # - after restoring from cache, there will be a tar which we unpack and then delete
- if [ -f bootstrap.tar ]; then
- sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
- sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
- rm bootstrap.tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
+ sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v2 1/6] meta: Switch to mmdebstrap
2023-05-18 9:47 ` [PATCH v2 1/6] meta: Switch " Anton Mikanovich
@ 2023-05-20 4:50 ` MOESSBAUER, Felix
2023-05-20 9:23 ` Anton Mikanovich
0 siblings, 1 reply; 15+ messages in thread
From: MOESSBAUER, Felix @ 2023-05-20 4:50 UTC (permalink / raw)
To: amikan, isar-users; +Cc: ubely, Kiszka, Jan
On Thu, 2023-05-18 at 12:47 +0300, Anton Mikanovich wrote:
> From: Uladzimir Bely <ubely@ilbers.de>
>
> Instead of debootstrap, use mmdebstrap alternative.
>
> Internally, it uses apt and allows to debootstrap the distro from
> multiple repositories.
Hi,
does this solve the issue that we cannot bootstrap ubuntu > focal on a
debian < bookworm host? This is currently not supported, as some debs
use zstd compression which is only supported from dpkg >= 1.21 on.
I hope mmdebstrap can work around this by not relying on the host dpkg,
as otherwise very soon bootstrapping from non bookworm hosts will not
be possible anymore. However, looking at the dependencies of mmdebstrap
I'm not too optimistic about that.
PS: This is also another reason to use bookworm as a base for the kas-
container (and new ISAR CI).
Felix
>
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> meta-isar/conf/distro/ubuntu-focal.conf | 4 --
> .../isar-bootstrap/isar-bootstrap.inc | 57 ++++++++++-------
> --
> 2 files changed, 30 insertions(+), 31 deletions(-)
>
> diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-
> isar/conf/distro/ubuntu-focal.conf
> index 0cb6958d..be8c951c 100644
> --- a/meta-isar/conf/distro/ubuntu-focal.conf
> +++ b/meta-isar/conf/distro/ubuntu-focal.conf
> @@ -17,10 +17,6 @@ BOOTSTRAP_KEY =
> "file://${LAYERDIR_isar}/conf/distro/ubuntu.public.key;sha256sum
> DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
> HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
>
> -
> -# that is what debootstrap_1.0.118ubuntu1 does anyways
> -DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
> -
> DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
>
> DISTRO_KERNELS ?= " \
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index b94ae0bd..f89c284e 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -15,7 +15,7 @@ PV = "1.0"
>
> BOOTSTRAP_FOR_HOST ?= "0"
>
> -DEBOOTSTRAP ?= "qemu-debootstrap"
> +DEBOOTSTRAP = "mmdebstrap"
> ROOTFSDIR = "${WORKDIR}/rootfs"
> APTPREFS = "${WORKDIR}/apt-preferences"
> APTSRCS = "${WORKDIR}/apt-sources"
> @@ -23,12 +23,14 @@ APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
> DISTRO_BOOTSTRAP_KEYFILES = ""
> THIRD_PARTY_APT_KEYFILES = ""
> DEPLOY_ISAR_BOOTSTRAP ?= ""
> -DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
> DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
> DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-
> certificates"
> DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST')
> == '1' else ''}"
> BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
> BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
> +BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
> +BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
> FILESEXTRAPATHS:append = ":${BBPATH}"
>
> inherit deb-dl-dir
> @@ -270,7 +272,7 @@ do_bootstrap[vardeps] += " \
> ISAR_ENABLE_COMPAT_ARCH \
> ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> "
> -do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
>
> @@ -286,27 +288,31 @@ do_bootstrap() {
> if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> debootstrap_args="$debootstrap_args --
> keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> fi
> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z
> "${BASE_REPO_KEY}" ]; then
> - debootstrap_args="$debootstrap_args --no-check-gpg"
> - fi
> E="${@ isar_export_proxies(d)}"
> - export BOOTSTRAP_FOR_HOST debootstrap_args E
> + export BOOTSTRAP_FOR_HOST
>
> - sudo rm -rf --one-file-system "${ROOTFSDIR}"
> deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> + sudo rm -rf --one-file-system "${ROOTFSDIR}"
> + mkdir -p "${ROOTFSDIR}"
> +
> + if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> + export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> + fi
> +
> + arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> +
> + sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP}
> $debootstrap_args \
> + $arch_param \
> + --mode=unshare \
> + ${@get_distro_components_argument(d)} \
> + "${@get_distro_suite(d)}" \
> + "${WORKDIR}/rootfs.tar.zst" \
> + "${@get_distro_source(d)}"
>
> sudo -E -s <<'EOSUDO'
> set -e
> - if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
> - arch_param="--arch=${DISTRO_ARCH}"
> - fi
> - ${DEBOOTSTRAP} $debootstrap_args \
> - $arch_param \
> - ${@get_distro_components_argument(d)} \
> - "${@get_distro_suite(d)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d)}" \
> - ${DISTRO_DEBOOTSTRAP_SCRIPT}
> +
> + tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --
> exclude="./dev/console"
>
> # Install apt config
> mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> @@ -409,19 +415,16 @@ SSTATEPOSTINSTFUNCS +=
> "bootstrap_sstate_finalize"
>
> bootstrap_sstate_prepare() {
> # this runs in SSTATE_BUILDDIR, which will be deleted
> automatically
> - lopts="--one-file-system --exclude=var/cache/apt/archives"
> - sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts
> $(basename "${ROOTFSDIR}")
> - sudo chown $(id -u):$(id -g) bootstrap.tar
> + sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> ./bootstrap.tar.zst
> + sudo chown $(id -u):$(id -g) bootstrap.tar.zst
> }
>
> bootstrap_sstate_finalize() {
> # this runs in SSTATE_INSTDIR
> - # - after building the bootstrap, the tar won't be there, but we
> also don't need to unpack
> - # - after restoring from cache, there will be a tar which we
> unpack and then delete
> - if [ -f bootstrap.tar ]; then
> - sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
> - sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> - rm bootstrap.tar
> + if [ -f bootstrap.tar.zst ]; then
> + mv bootstrap.tar.zst "$(dirname
> "${ROOTFSDIR}")/rootfs.tar.zst"
> + sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> + "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
> fi
> }
>
> --
> 2.34.1
>
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v2 1/6] meta: Switch to mmdebstrap
2023-05-20 4:50 ` MOESSBAUER, Felix
@ 2023-05-20 9:23 ` Anton Mikanovich
2023-05-21 6:38 ` Jan Kiszka
0 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-20 9:23 UTC (permalink / raw)
To: MOESSBAUER, Felix, isar-users; +Cc: ubely, Kiszka, Jan
20/05/2023 07:50, MOESSBAUER, Felix wrote:
> Hi,
>
> does this solve the issue that we cannot bootstrap ubuntu > focal on a
> debian < bookworm host? This is currently not supported, as some debs
> use zstd compression which is only supported from dpkg >= 1.21 on.
>
> I hope mmdebstrap can work around this by not relying on the host dpkg,
> as otherwise very soon bootstrapping from non bookworm hosts will not
> be possible anymore. However, looking at the dependencies of mmdebstrap
> I'm not too optimistic about that.
>
> PS: This is also another reason to use bookworm as a base for the kas-
> container (and new ISAR CI).
>
> Felix
Hello Felix,
I've just tried to build jammy on bullseye host with mmdebstrap and
unfortunately it has the same issue.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH v2 1/6] meta: Switch to mmdebstrap
2023-05-20 9:23 ` Anton Mikanovich
@ 2023-05-21 6:38 ` Jan Kiszka
0 siblings, 0 replies; 15+ messages in thread
From: Jan Kiszka @ 2023-05-21 6:38 UTC (permalink / raw)
To: Anton Mikanovich, MOESSBAUER, Felix, isar-users; +Cc: ubely
On 20.05.23 11:23, Anton Mikanovich wrote:
> 20/05/2023 07:50, MOESSBAUER, Felix wrote:
>> Hi,
>>
>> does this solve the issue that we cannot bootstrap ubuntu > focal on a
>> debian < bookworm host? This is currently not supported, as some debs
>> use zstd compression which is only supported from dpkg >= 1.21 on.
>>
>> I hope mmdebstrap can work around this by not relying on the host dpkg,
>> as otherwise very soon bootstrapping from non bookworm hosts will not
>> be possible anymore. However, looking at the dependencies of mmdebstrap
>> I'm not too optimistic about that.
>>
>> PS: This is also another reason to use bookworm as a base for the kas-
>> container (and new ISAR CI).
>>
>> Felix
>
> Hello Felix,
> I've just tried to build jammy on bullseye host with mmdebstrap and
> unfortunately it has the same issue.
>
AFAIK, it will take dpkg from bookworm to add zstd support which is
needed for Ubuntu after focal.
Jan
--
Siemens AG, Technology
Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v2 2/6] isar-bootstrap: Move preparations to hooks
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 1/6] meta: Switch " Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 3/6] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
` (4 subsequent siblings)
6 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-bootstrap/isar-bootstrap.inc | 196 +++++++++---------
1 file changed, 102 insertions(+), 94 deletions(-)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index f89c284e..fba876ea 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -212,24 +212,21 @@ def get_distro_components_argument(d):
else:
return ""
-APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
-do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
-do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
+do_generate_keyrings[dirs] = "${DEBDIR}"
do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
do_generate_keyrings[network] = "${TASK_USE_SUDO}"
do_generate_keyrings() {
if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
- chmod 777 "${APT_KEYS_DIR}"
for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
done
fi
if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
done
fi
}
@@ -272,12 +269,14 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
inherit compat
+DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -287,124 +286,133 @@ do_bootstrap() {
debootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST
- deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${ROOTFSDIR}"
mkdir -p "${ROOTFSDIR}"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
+ bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
+
+ # no need to sync /var/cache/apt/archives if base-apt used
+ syncin='echo skip sync-in'
+ syncout='echo skip sync-out'
+ extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
+ extra_extract="$syncout"
+ # save mmdebstrap tempdir for cleanup
+ extra_essential="mkdir -p \$1/$base_apt_tmp && \
+ echo \$1 > ${WORKDIR}/mmtmpdir && \
+ mount --bind '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
+ # replace base-apt mount in tmp with /base-apt mount
+ extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
+ \$1/etc/apt/sources.list.d/*.list && \
+ mkdir -p \$1/base-apt && \
+ mount --bind '${REPO_BASE_DIR}' \$1/base-apt && \
+ chroot \$1 apt-get update -y \
+ -o APT::Update::Error-Mode=any && \
+ umount \$1/base-apt && \
+ umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
+ umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
+ else
+ deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
+ install -v -m644 "${APTSRCS}" \
+ "${WORKDIR}/sources.list.d/bootstrap.list"
+
+ syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
+ "$1/var/cache/apt/archives/" || true'
+ syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "$1/var/cache/apt/archives/"*.deb \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
+ extra_setup="$syncin"
+ extra_extract="$syncout"
+ # prefetch apt debs because mmdebstrap will clean them on next stage
+ extra_essential='apt-get install apt -y -d \
+ -o Dir::State="$1/var/lib/apt" \
+ -o Dir::Etc="$1/etc/apt" \
+ -o Dir::Cache="$1/var/cache/apt" \
+ -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
+ extra_essential="$extra_essential && $syncout"
+ extra_customize="$syncout"
+ fi
+
if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
fi
arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
+ fi
+
+ # Cleanup mounts if fails
+ trap 'exit 1' INT HUP QUIT TERM ALRM USR1
+ trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
+ && rm "${WORKDIR}/mmtmpdir"; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
+ && sudo umount $tmpdir/$base_apt_tmp; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
+ && sudo umount $tmpdir/base-apt; \
+ [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
+ [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
+ && sudo umount $base_apt_tmp \
+ && rm -rf --one-file-system $base_apt_tmp' EXIT
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
$arch_param \
--mode=unshare \
+ --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
+ --setup-hook="$extra_setup" \
+ --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
+ --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
+ --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
+ --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
+ --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
+ --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
+ --extract-hook="$extra_extract" \
+ --essential-hook="$extra_essential" \
+ --customize-hook="$extra_customize" \
+ --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
+ --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
+ --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
+ --skip=cleanup/apt \
+ --skip=download/empty \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
- "${@get_distro_source(d)}"
+ "$bootstrap_list"
sudo -E -s <<'EOSUDO'
set -e
tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
- # Install apt config
- mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
-
- mkdir -p ${ROOTFSDIR}/base-apt
- mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- else
- install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- fi
- install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
- rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
- find ${APT_KEYS_DIR}/ -type f | while read keyfile
- do
- MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
- echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
- export GNUPGHOME="${MY_GPGHOME}"
- APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
-
- kfn="$(basename $keyfile)"
- cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
- chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
- --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
- rm "${ROOTFSDIR}/tmp/$kfn"
-
- echo "Removing ${MY_GPGHOME}"
- rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
- done
-
- # Set locale
- install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
-
- sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-
# setup chroot
install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
"${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
- # update APT
- mount -o bind,private /dev ${ROOTFSDIR}/dev
- mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
- mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
- mount -t proc none ${ROOTFSDIR}/proc
- mount --rbind /sys ${ROOTFSDIR}/sys
- mount --make-rslave ${ROOTFSDIR}/sys
-
- export DEBIAN_FRONTEND=noninteractive
-
- if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
- fi
-
- if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
- fi
-
- chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
- -o APT::Update::Error-Mode=any
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
- chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
- -o Debug::pkgProblemResolver=yes
-
- umount -l "${ROOTFSDIR}/dev/shm"
- umount -l "${ROOTFSDIR}/dev/pts"
- umount -l "${ROOTFSDIR}/dev"
- umount -l "${ROOTFSDIR}/proc"
- umount -l "${ROOTFSDIR}/sys"
- umount -l "${ROOTFSDIR}/base-apt" || true
-
# Finalize debootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
- deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
-
- # Cleanup apt cache
- sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
+ deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
+ fi
}
addtask bootstrap before do_build after do_generate_keyrings
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v2 3/6] isar-bootstrap: Use tar output instead of directory
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 1/6] meta: Switch " Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 2/6] isar-bootstrap: Move preparations to hooks Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 4/6] user_manual.md: Replace debootstrap with mmdebstrap Anton Mikanovich
` (3 subsequent siblings)
6 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Do not unpack tar with rootfs content inside bootstrap recipe.
This also requires moving chroot prepare step to the later unpack step.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/rootfs.bbclass | 11 +++++----
.../isar-bootstrap/isar-bootstrap.inc | 23 ++++---------------
2 files changed, 12 insertions(+), 22 deletions(-)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 22449d71..29d5fd53 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -73,12 +73,15 @@ rootfs_do_qemu() {
fi
}
-BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
-BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
+BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
rootfs_prepare[weight] = "25"
rootfs_prepare(){
- sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+ sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # setup chroot
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
}
ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -222,7 +225,7 @@ cache_deb_src() {
# Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
# relies on that through out the build. Copy that state information instead of apt-get update
# which generates a new state from upstream.
- sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+ sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index fba876ea..2aeceb58 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -16,7 +16,6 @@ PV = "1.0"
BOOTSTRAP_FOR_HOST ?= "0"
DEBOOTSTRAP = "mmdebstrap"
-ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
@@ -290,9 +289,6 @@ do_bootstrap() {
fi
E="${@ isar_export_proxies(d)}"
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
- mkdir -p "${ROOTFSDIR}"
-
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
@@ -397,18 +393,9 @@ do_bootstrap() {
"${WORKDIR}/rootfs.tar.zst" \
"$bootstrap_list"
- sudo -E -s <<'EOSUDO'
- set -e
-
- tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
-
- # setup chroot
- install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+ # Finalize debootstrap by setting the link in deploy
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
- # Finalize debootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
-EOSUDO
if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
@@ -423,15 +410,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
if [ -f bootstrap.tar.zst ]; then
- mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
- sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
"${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v2 4/6] user_manual.md: Replace debootstrap with mmdebstrap
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
` (2 preceding siblings ...)
2023-05-18 9:47 ` [PATCH v2 3/6] isar-bootstrap: Use tar output instead of directory Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-19 4:55 ` Jan Kiszka
2023-05-18 9:47 ` [PATCH v2 5/6] isar-bootstrap: Remove unused code Anton Mikanovich
` (2 subsequent siblings)
6 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
---
doc/user_manual.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index 60b7ca92..999cbd8b 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -65,7 +65,9 @@ Install the following packages:
apt install \
binfmt-support \
bzip2 \
- debootstrap \
+ mmdebstrap \
+ arch-test \
+ apt-utils \
dosfstools \
dpkg-dev \
gettext-base \
@@ -128,7 +130,7 @@ apt install qemu
### Setup Sudo
-Isar requires `sudo` rights without password to work with `chroot` and `debootstrap`. To add them, use the following steps:
+Isar requires `sudo` rights without password to work with `chroot`. To add them, use the following steps:
```
# visudo
```
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v2 4/6] user_manual.md: Replace debootstrap with mmdebstrap
2023-05-18 9:47 ` [PATCH v2 4/6] user_manual.md: Replace debootstrap with mmdebstrap Anton Mikanovich
@ 2023-05-19 4:55 ` Jan Kiszka
2023-05-19 6:14 ` Anton Mikanovich
0 siblings, 1 reply; 15+ messages in thread
From: Jan Kiszka @ 2023-05-19 4:55 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
Missing signed-off - due to missing commit message.
Jan
On 18.05.23 11:47, Anton Mikanovich wrote:
> ---
> doc/user_manual.md | 6 ++++--
> 1 file changed, 4 insertions(+), 2 deletions(-)
>
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index 60b7ca92..999cbd8b 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -65,7 +65,9 @@ Install the following packages:
> apt install \
> binfmt-support \
> bzip2 \
> - debootstrap \
> + mmdebstrap \
> + arch-test \
> + apt-utils \
> dosfstools \
> dpkg-dev \
> gettext-base \
> @@ -128,7 +130,7 @@ apt install qemu
>
> ### Setup Sudo
>
> -Isar requires `sudo` rights without password to work with `chroot` and `debootstrap`. To add them, use the following steps:
> +Isar requires `sudo` rights without password to work with `chroot`. To add them, use the following steps:
> ```
> # visudo
> ```
--
Siemens AG, Technology
Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v2 5/6] isar-bootstrap: Remove unused code
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
` (3 preceding siblings ...)
2023-05-18 9:47 ` [PATCH v2 4/6] user_manual.md: Replace debootstrap with mmdebstrap Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-18 9:47 ` [PATCH v2 6/6] CI: Install mmdebstrap Anton Mikanovich
2023-05-19 4:57 ` [PATCH v2 0/6] Migrate to mmdebstrap Jan Kiszka
6 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Anton Mikanovich
Some external logic like installing gpg and ca-certificates is now
doing by mmdebstrap inself. So remove duplications.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-bootstrap/isar-bootstrap.inc | 26 -------------------
1 file changed, 26 deletions(-)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2aeceb58..b52edc05 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -23,8 +23,6 @@ DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
@@ -177,30 +175,6 @@ def get_distro_primary_source_entry(d):
return source[2:]
bb.fatal('Invalid apt sources list')
-def get_distro_have_https_source(d):
- return any(source[2].startswith("https://") for source in generate_distro_sources(d))
-
-def get_distro_needs_https_support(d):
- if get_distro_have_https_source(d):
- return "https-support"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
-
-def get_distro_needs_gpg_support(d):
- if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
- d.getVar("THIRD_PARTY_APT_KEYS") or \
- d.getVar("BASE_REPO_KEY"):
- return "gnupg"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-
-def get_distro_source(d):
- return get_distro_primary_source_entry(d)[0]
-
def get_distro_suite(d):
return get_distro_primary_source_entry(d)[1]
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* [PATCH v2 6/6] CI: Install mmdebstrap
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
` (4 preceding siblings ...)
2023-05-18 9:47 ` [PATCH v2 5/6] isar-bootstrap: Remove unused code Anton Mikanovich
@ 2023-05-18 9:47 ` Anton Mikanovich
2023-05-19 4:57 ` [PATCH v2 0/6] Migrate to mmdebstrap Jan Kiszka
6 siblings, 0 replies; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-18 9:47 UTC (permalink / raw)
To: isar-users; +Cc: Uladzimir Bely, Anton Mikanovich
From: Uladzimir Bely <ubely@ilbers.de>
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
scripts/ci_build.sh | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 2fa5cccd..055f9211 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -127,6 +127,12 @@ if echo "$TAGS" | grep -Fqive "-startvm"; then
fi
fi
+# install mmdebstrap
+if ! command -v mmdebstrap > /dev/null; then
+ sudo apt-get update -qq
+ sudo apt-get install -y mmdebstrap
+fi
+
# Provide working path
mkdir -p .config/avocado
cat <<EOF > .config/avocado/avocado.conf
--
2.34.1
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v2 0/6] Migrate to mmdebstrap
2023-05-18 9:47 [PATCH v2 0/6] Migrate to mmdebstrap Anton Mikanovich
` (5 preceding siblings ...)
2023-05-18 9:47 ` [PATCH v2 6/6] CI: Install mmdebstrap Anton Mikanovich
@ 2023-05-19 4:57 ` Jan Kiszka
2023-05-19 6:13 ` Anton Mikanovich
6 siblings, 1 reply; 15+ messages in thread
From: Jan Kiszka @ 2023-05-19 4:57 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
On 18.05.23 11:47, Anton Mikanovich wrote:
> Switch Isar from debootstrap to mmdebstrap.
> It makes isar-bootstrap target ~10% faster on both cross and native rootfs
> types. Also mmdebstrap support using SOURCE_DATE_EPOCH for images reproducibly
> out of the box. But the main advantage is an ability to remove sudo for
> debootstrapping later on.
> This change brakes Debian Buster support as host distro.
Where is this last aspect described in the patches?
Jan
>
> Changes since v2:
> - Move mmdebstrap tmpdir to workdir
> - Fix kas building
> - Improve mounts cleanup
>
> Anton Mikanovich (4):
> isar-bootstrap: Move preparations to hooks
> isar-bootstrap: Use tar output instead of directory
> user_manual.md: Replace debootstrap with mmdebstrap
> isar-bootstrap: Remove unused code
>
> Uladzimir Bely (2):
> meta: Switch to mmdebstrap
> CI: Install mmdebstrap
>
> doc/user_manual.md | 6 +-
> meta-isar/conf/distro/ubuntu-focal.conf | 4 -
> meta/classes/rootfs.bbclass | 11 +-
> .../isar-bootstrap/isar-bootstrap.inc | 274 ++++++++----------
> scripts/ci_build.sh | 6 +
> 5 files changed, 140 insertions(+), 161 deletions(-)
>
--
Siemens AG, Technology
Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v2 0/6] Migrate to mmdebstrap
2023-05-19 4:57 ` [PATCH v2 0/6] Migrate to mmdebstrap Jan Kiszka
@ 2023-05-19 6:13 ` Anton Mikanovich
2023-05-19 12:55 ` Jan Kiszka
0 siblings, 1 reply; 15+ messages in thread
From: Anton Mikanovich @ 2023-05-19 6:13 UTC (permalink / raw)
To: Jan Kiszka, isar-users
19/05/2023 07:57, Jan Kiszka wrote:
> On 18.05.23 11:47, Anton Mikanovich wrote:
>> Switch Isar from debootstrap to mmdebstrap.
>> It makes isar-bootstrap target ~10% faster on both cross and native rootfs
>> types. Also mmdebstrap support using SOURCE_DATE_EPOCH for images reproducibly
>> out of the box. But the main advantage is an ability to remove sudo for
>> debootstrapping later on.
>> This change brakes Debian Buster support as host distro.
> Where is this last aspect described in the patches?
>
> Jan
Hello Jan,
It's all about the usage of mmdebstrap itself: buster version of the
package is
too old to have all the features already used in isar-bootstrap recipe.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH v2 0/6] Migrate to mmdebstrap
2023-05-19 6:13 ` Anton Mikanovich
@ 2023-05-19 12:55 ` Jan Kiszka
0 siblings, 0 replies; 15+ messages in thread
From: Jan Kiszka @ 2023-05-19 12:55 UTC (permalink / raw)
To: Anton Mikanovich, isar-users
On 19.05.23 08:13, Anton Mikanovich wrote:
> 19/05/2023 07:57, Jan Kiszka wrote:
>> On 18.05.23 11:47, Anton Mikanovich wrote:
>>> Switch Isar from debootstrap to mmdebstrap.
>>> It makes isar-bootstrap target ~10% faster on both cross and native
>>> rootfs
>>> types. Also mmdebstrap support using SOURCE_DATE_EPOCH for images
>>> reproducibly
>>> out of the box. But the main advantage is an ability to remove sudo for
>>> debootstrapping later on.
>>> This change brakes Debian Buster support as host distro.
>> Where is this last aspect described in the patches?
>>
>> Jan
>
> Hello Jan,
> It's all about the usage of mmdebstrap itself: buster version of the
> package is
> too old to have all the features already used in isar-bootstrap recipe.
>
I meant that our user guide still states that host >= buster, even after
your patches.
Jan
--
Siemens AG, Technology
Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 15+ messages in thread