* [PATCH v4 1/6] meta: Switch to mmdebstrap
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
2023-09-27 10:02 ` Jan Kiszka
2023-09-27 4:33 ` [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks Uladzimir Bely
` (4 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
Instead of debootstrap, use mmdebstrap alternative.
Internally, it uses apt and allows to debootstrap the distro from
multiple repositories.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta-isar/conf/distro/ubuntu-common.inc | 4 --
.../isar-bootstrap/isar-bootstrap.inc | 57 ++++++++++---------
2 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/meta-isar/conf/distro/ubuntu-common.inc b/meta-isar/conf/distro/ubuntu-common.inc
index 6a990a89..afe96b73 100644
--- a/meta-isar/conf/distro/ubuntu-common.inc
+++ b/meta-isar/conf/distro/ubuntu-common.inc
@@ -18,10 +18,6 @@ HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
# kernel package name is linux-image-generic in Ubuntu
KERNEL_NAME = "generic"
-
-# that is what debootstrap_1.0.118ubuntu1 does anyways
-DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
-
DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
DISTRO_KERNELS ?= " \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 3477c2fb..637018c6 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -15,7 +15,7 @@ PV = "1.0"
BOOTSTRAP_FOR_HOST ?= "0"
-DEBOOTSTRAP ?= "qemu-debootstrap"
+DEBOOTSTRAP = "mmdebstrap"
ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
@@ -23,12 +23,14 @@ APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
-DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
+DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
+BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
+BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
FILESEXTRAPATHS:append = ":${BBPATH}"
inherit deb-dl-dir
@@ -267,7 +269,7 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
@@ -283,27 +285,31 @@ do_bootstrap() {
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
fi
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z "${BASE_REPO_KEY}" ]; then
- debootstrap_args="$debootstrap_args --no-check-gpg"
- fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST debootstrap_args E
+ export BOOTSTRAP_FOR_HOST
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${ROOTFSDIR}"
+ mkdir -p "${ROOTFSDIR}"
+
+ if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
+ export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
+ fi
+
+ arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+
+ sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
+ $arch_param \
+ --mode=unshare \
+ ${@get_distro_components_argument(d)} \
+ "${@get_distro_suite(d)}" \
+ "${WORKDIR}/rootfs.tar.zst" \
+ "${@get_distro_source(d)}"
sudo -E -s <<'EOSUDO'
set -e
- if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
- arch_param="--arch=${DISTRO_ARCH}"
- fi
- ${DEBOOTSTRAP} $debootstrap_args \
- $arch_param \
- ${@get_distro_components_argument(d)} \
- "${@get_distro_suite(d)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d)}" \
- ${DISTRO_DEBOOTSTRAP_SCRIPT}
+
+ tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
# Install apt config
mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
@@ -406,19 +412,16 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- lopts="--one-file-system --exclude=var/cache/apt/archives"
- sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
- sudo chown $(id -u):$(id -g) bootstrap.tar
+ sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
- # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
- # - after restoring from cache, there will be a tar which we unpack and then delete
- if [ -f bootstrap.tar ]; then
- sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
- sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
- rm bootstrap.tar
+ if [ -f bootstrap.tar.zst ]; then
+ mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
+ sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v4 1/6] meta: Switch to mmdebstrap
2023-09-27 4:33 ` [PATCH v4 1/6] meta: Switch " Uladzimir Bely
@ 2023-09-27 10:02 ` Jan Kiszka
0 siblings, 0 replies; 11+ messages in thread
From: Jan Kiszka @ 2023-09-27 10:02 UTC (permalink / raw)
To: Uladzimir Bely, isar-users
On 27.09.23 06:33, Uladzimir Bely wrote:
> Instead of debootstrap, use mmdebstrap alternative.
>
> Internally, it uses apt and allows to debootstrap the distro from
> multiple repositories.
>
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> meta-isar/conf/distro/ubuntu-common.inc | 4 --
> .../isar-bootstrap/isar-bootstrap.inc | 57 ++++++++++---------
> 2 files changed, 30 insertions(+), 31 deletions(-)
>
> diff --git a/meta-isar/conf/distro/ubuntu-common.inc b/meta-isar/conf/distro/ubuntu-common.inc
> index 6a990a89..afe96b73 100644
> --- a/meta-isar/conf/distro/ubuntu-common.inc
> +++ b/meta-isar/conf/distro/ubuntu-common.inc
> @@ -18,10 +18,6 @@ HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
>
> # kernel package name is linux-image-generic in Ubuntu
> KERNEL_NAME = "generic"
> -
> -# that is what debootstrap_1.0.118ubuntu1 does anyways
> -DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
> -
> DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
>
> DISTRO_KERNELS ?= " \
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 3477c2fb..637018c6 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -15,7 +15,7 @@ PV = "1.0"
>
> BOOTSTRAP_FOR_HOST ?= "0"
>
> -DEBOOTSTRAP ?= "qemu-debootstrap"
> +DEBOOTSTRAP = "mmdebstrap"
Is it still useful to have this variable, or isn't the code including
mmdebstrap assumptions anyway?
> ROOTFSDIR = "${WORKDIR}/rootfs"
> APTPREFS = "${WORKDIR}/apt-preferences"
> APTSRCS = "${WORKDIR}/apt-sources"
> @@ -23,12 +23,14 @@ APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
> DISTRO_BOOTSTRAP_KEYFILES = ""
> THIRD_PARTY_APT_KEYFILES = ""
> DEPLOY_ISAR_BOOTSTRAP ?= ""
> -DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
> DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
> DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
> DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
> BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
> BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
> +BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
> +BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
> FILESEXTRAPATHS:append = ":${BBPATH}"
>
> inherit deb-dl-dir
> @@ -267,7 +269,7 @@ do_bootstrap[vardeps] += " \
> ISAR_ENABLE_COMPAT_ARCH \
> ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> "
> -do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
>
> @@ -283,27 +285,31 @@ do_bootstrap() {
> if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> fi
> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z "${BASE_REPO_KEY}" ]; then
> - debootstrap_args="$debootstrap_args --no-check-gpg"
> - fi
> E="${@ isar_export_proxies(d)}"
> - export BOOTSTRAP_FOR_HOST debootstrap_args E
> + export BOOTSTRAP_FOR_HOST
>
> - sudo rm -rf --one-file-system "${ROOTFSDIR}"
> deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> + sudo rm -rf --one-file-system "${ROOTFSDIR}"
> + mkdir -p "${ROOTFSDIR}"
> +
> + if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> + export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> + fi
> +
> + arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> +
> + sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
"debootstrap_args"? Should probably be renamed.
> + $arch_param \
> + --mode=unshare \
> + ${@get_distro_components_argument(d)} \
> + "${@get_distro_suite(d)}" \
> + "${WORKDIR}/rootfs.tar.zst" \
> + "${@get_distro_source(d)}"
>
> sudo -E -s <<'EOSUDO'
> set -e
> - if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
> - arch_param="--arch=${DISTRO_ARCH}"
> - fi
> - ${DEBOOTSTRAP} $debootstrap_args \
> - $arch_param \
> - ${@get_distro_components_argument(d)} \
> - "${@get_distro_suite(d)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d)}" \
> - ${DISTRO_DEBOOTSTRAP_SCRIPT}
> +
> + tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
>
> # Install apt config
> mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> @@ -406,19 +412,16 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>
> bootstrap_sstate_prepare() {
> # this runs in SSTATE_BUILDDIR, which will be deleted automatically
> - lopts="--one-file-system --exclude=var/cache/apt/archives"
> - sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts $(basename "${ROOTFSDIR}")
> - sudo chown $(id -u):$(id -g) bootstrap.tar
> + sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
> + sudo chown $(id -u):$(id -g) bootstrap.tar.zst
> }
>
> bootstrap_sstate_finalize() {
> # this runs in SSTATE_INSTDIR
> - # - after building the bootstrap, the tar won't be there, but we also don't need to unpack
> - # - after restoring from cache, there will be a tar which we unpack and then delete
> - if [ -f bootstrap.tar ]; then
> - sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
> - sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> - rm bootstrap.tar
> + if [ -f bootstrap.tar.zst ]; then
> + mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> + sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> + "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
> fi
> }
>
Jan
--
Siemens AG, Technology
Linux Expert Center
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 1/6] meta: Switch " Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
2023-09-27 10:05 ` Jan Kiszka
2023-09-27 4:33 ` [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory Uladzimir Bely
` (3 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
From: Anton Mikanovich <amikan@ilbers.de>
Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-bootstrap/isar-bootstrap.inc | 196 +++++++++---------
1 file changed, 102 insertions(+), 94 deletions(-)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 637018c6..d8ac9d00 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -211,24 +211,21 @@ def get_distro_components_argument(d):
else:
return ""
-APT_KEYS_DIR = "${WORKDIR}/aptkeys"
DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
-do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
-do_generate_keyrings[dirs] = "${DL_DIR}"
+do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
+do_generate_keyrings[dirs] = "${DEBDIR}"
do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
do_generate_keyrings[network] = "${TASK_USE_SUDO}"
do_generate_keyrings() {
if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
- chmod 777 "${APT_KEYS_DIR}"
for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
+ sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
done
fi
if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
- cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
done
fi
}
@@ -269,12 +266,14 @@ do_bootstrap[vardeps] += " \
ISAR_ENABLE_COMPAT_ARCH \
${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
"
-do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
+do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
inherit compat
+DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
+
do_bootstrap() {
if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
@@ -284,124 +283,133 @@ do_bootstrap() {
debootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
+ cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
fi
E="${@ isar_export_proxies(d)}"
- export BOOTSTRAP_FOR_HOST
- deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${ROOTFSDIR}"
mkdir -p "${ROOTFSDIR}"
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
+ base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
+ bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
+ line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
+
+ # no need to sync /var/cache/apt/archives if base-apt used
+ syncin='echo skip sync-in'
+ syncout='echo skip sync-out'
+ extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
+ extra_extract="$syncout"
+ # save mmdebstrap tempdir for cleanup
+ extra_essential="mkdir -p \$1/$base_apt_tmp && \
+ echo \$1 > ${WORKDIR}/mmtmpdir && \
+ mount --bind '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
+ # replace base-apt mount in tmp with /base-apt mount
+ extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
+ \$1/etc/apt/sources.list.d/*.list && \
+ mkdir -p \$1/base-apt && \
+ mount --bind '${REPO_BASE_DIR}' \$1/base-apt && \
+ chroot \$1 apt-get update -y \
+ -o APT::Update::Error-Mode=any && \
+ umount \$1/base-apt && \
+ umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
+ umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
+ else
+ deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+
+ bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
+ install -v -m644 "${APTSRCS}" \
+ "${WORKDIR}/sources.list.d/bootstrap.list"
+
+ syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
+ "$1/var/cache/apt/archives/" || true'
+ syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
+ "$1/var/cache/apt/archives/"*.deb \
+ "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
+ extra_setup="$syncin"
+ extra_extract="$syncout"
+ # prefetch apt debs because mmdebstrap will clean them on next stage
+ extra_essential='apt-get install apt -y -d \
+ -o Dir::State="$1/var/lib/apt" \
+ -o Dir::Etc="$1/etc/apt" \
+ -o Dir::Cache="$1/var/cache/apt" \
+ -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
+ extra_essential="$extra_essential && $syncout"
+ extra_customize="$syncout"
+ fi
+
if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
fi
arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
+ if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
+ arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
+ fi
+
+ # Cleanup mounts if fails
+ trap 'exit 1' INT HUP QUIT TERM ALRM USR1
+ trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
+ && rm "${WORKDIR}/mmtmpdir"; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
+ && sudo umount $tmpdir/$base_apt_tmp; \
+ [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
+ && sudo umount $tmpdir/base-apt; \
+ [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
+ [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
+ && sudo umount $base_apt_tmp \
+ && rm -rf --one-file-system $base_apt_tmp' EXIT
sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
$arch_param \
--mode=unshare \
+ --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
+ --setup-hook="$extra_setup" \
+ --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
+ --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
+ --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
+ --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
+ --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
+ --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
+ --extract-hook="$extra_extract" \
+ --essential-hook="$extra_essential" \
+ --customize-hook="$extra_customize" \
+ --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
+ --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
+ --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
+ --skip=cleanup/apt \
+ --skip=download/empty \
${@get_distro_components_argument(d)} \
"${@get_distro_suite(d)}" \
"${WORKDIR}/rootfs.tar.zst" \
- "${@get_distro_source(d)}"
+ "$bootstrap_list"
sudo -E -s <<'EOSUDO'
set -e
tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
- # Install apt config
- mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
- if [ -z "${BASE_REPO_KEY}" ]; then
- line="[trusted=yes] ${line}"
- fi
- echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
-
- mkdir -p ${ROOTFSDIR}/base-apt
- mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- else
- install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- fi
- install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
- rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
- find ${APT_KEYS_DIR}/ -type f | while read keyfile
- do
- MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
- echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
- export GNUPGHOME="${MY_GPGHOME}"
- APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
-
- kfn="$(basename $keyfile)"
- cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
- chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
- --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
- rm "${ROOTFSDIR}/tmp/$kfn"
-
- echo "Removing ${MY_GPGHOME}"
- rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
- done
-
- # Set locale
- install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
-
- sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-
# setup chroot
install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
"${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
- # update APT
- mount -o bind,private /dev ${ROOTFSDIR}/dev
- mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
- mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
- mount -t proc none ${ROOTFSDIR}/proc
- mount --rbind /sys ${ROOTFSDIR}/sys
- mount --make-rslave ${ROOTFSDIR}/sys
-
- export DEBIAN_FRONTEND=noninteractive
-
- if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
- fi
-
- if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
- chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
- fi
-
- chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
- -o APT::Update::Error-Mode=any
- chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
- chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
- -o Debug::pkgProblemResolver=yes
-
- umount -l "${ROOTFSDIR}/dev/shm"
- umount -l "${ROOTFSDIR}/dev/pts"
- umount -l "${ROOTFSDIR}/dev"
- umount -l "${ROOTFSDIR}/proc"
- umount -l "${ROOTFSDIR}/sys"
- umount -l "${ROOTFSDIR}/base-apt" || true
-
# Finalize debootstrap by setting the link in deploy
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
- deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
-
- # Cleanup apt cache
- sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
+ if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
+ deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
+ sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
+ fi
}
addtask bootstrap before do_build after do_generate_keyrings
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks
2023-09-27 4:33 ` [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks Uladzimir Bely
@ 2023-09-27 10:05 ` Jan Kiszka
0 siblings, 0 replies; 11+ messages in thread
From: Jan Kiszka @ 2023-09-27 10:05 UTC (permalink / raw)
To: Uladzimir Bely, isar-users
On 27.09.23 06:33, Uladzimir Bely wrote:
> From: Anton Mikanovich <amikan@ilbers.de>
>
> Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> .../isar-bootstrap/isar-bootstrap.inc | 196 +++++++++---------
> 1 file changed, 102 insertions(+), 94 deletions(-)
>
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 637018c6..d8ac9d00 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -211,24 +211,21 @@ def get_distro_components_argument(d):
> else:
> return ""
>
> -APT_KEYS_DIR = "${WORKDIR}/aptkeys"
> DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
>
> -do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
> -do_generate_keyrings[dirs] = "${DL_DIR}"
> +do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
> +do_generate_keyrings[dirs] = "${DEBDIR}"
> do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
> do_generate_keyrings[network] = "${TASK_USE_SUDO}"
> do_generate_keyrings() {
> if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
> - chmod 777 "${APT_KEYS_DIR}"
> for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
> - cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> + sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
Isn't apt-key deprecated? Why re-introducing it then?
> done
> fi
> if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
> for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
> sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
> - cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> done
> fi
> }
> @@ -269,12 +266,14 @@ do_bootstrap[vardeps] += " \
> ISAR_ENABLE_COMPAT_ARCH \
> ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> "
> -do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
> do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
>
> inherit compat
>
> +DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
> +
> do_bootstrap() {
> if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
> @@ -284,124 +283,133 @@ do_bootstrap() {
> debootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
> if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> + cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
> fi
> E="${@ isar_export_proxies(d)}"
> - export BOOTSTRAP_FOR_HOST
>
> - deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> sudo rm -rf --one-file-system "${ROOTFSDIR}"
> mkdir -p "${ROOTFSDIR}"
>
> + if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> + base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
> + bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
> + line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
> + line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
> +
> + # no need to sync /var/cache/apt/archives if base-apt used
> + syncin='echo skip sync-in'
> + syncout='echo skip sync-out'
> + extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
> + extra_extract="$syncout"
> + # save mmdebstrap tempdir for cleanup
> + extra_essential="mkdir -p \$1/$base_apt_tmp && \
> + echo \$1 > ${WORKDIR}/mmtmpdir && \
> + mount --bind '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
> + # replace base-apt mount in tmp with /base-apt mount
> + extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
> + \$1/etc/apt/sources.list.d/*.list && \
> + mkdir -p \$1/base-apt && \
> + mount --bind '${REPO_BASE_DIR}' \$1/base-apt && \
> + chroot \$1 apt-get update -y \
> + -o APT::Update::Error-Mode=any && \
> + umount \$1/base-apt && \
> + umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
> + umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
> + else
> + deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> +
> + bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
> + install -v -m644 "${APTSRCS}" \
> + "${WORKDIR}/sources.list.d/bootstrap.list"
> +
> + syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
> + "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
> + "$1/var/cache/apt/archives/" || true'
> + syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
> + "$1/var/cache/apt/archives/"*.deb \
> + "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
> + extra_setup="$syncin"
> + extra_extract="$syncout"
> + # prefetch apt debs because mmdebstrap will clean them on next stage
> + extra_essential='apt-get install apt -y -d \
> + -o Dir::State="$1/var/lib/apt" \
> + -o Dir::Etc="$1/etc/apt" \
> + -o Dir::Cache="$1/var/cache/apt" \
> + -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
> + extra_essential="$extra_essential && $syncout"
> + extra_customize="$syncout"
> + fi
> +
> if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> fi
>
> arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> + if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> + arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> + fi
> +
> + # Cleanup mounts if fails
> + trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> + trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
> + && rm "${WORKDIR}/mmtmpdir"; \
> + [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
> + && sudo umount $tmpdir/$base_apt_tmp; \
> + [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
> + && sudo umount $tmpdir/base-apt; \
> + [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
> + [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
> + && sudo umount $base_apt_tmp \
> + && rm -rf --one-file-system $base_apt_tmp' EXIT
>
> sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
> $arch_param \
> --mode=unshare \
> + --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
> + --setup-hook="$extra_setup" \
> + --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
> + --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
> + --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
> + --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
> + --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
> + --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
> + --extract-hook="$extra_extract" \
> + --essential-hook="$extra_essential" \
> + --customize-hook="$extra_customize" \
> + --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
> + --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
> + --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
> + --skip=cleanup/apt \
> + --skip=download/empty \
> ${@get_distro_components_argument(d)} \
> "${@get_distro_suite(d)}" \
> "${WORKDIR}/rootfs.tar.zst" \
> - "${@get_distro_source(d)}"
> + "$bootstrap_list"
>
> sudo -E -s <<'EOSUDO'
> set -e
>
> tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
>
> - # Install apt config
> - mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> - install -v -m644 "${APTPREFS}" \
> - "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> - mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> - line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> - if [ -z "${BASE_REPO_KEY}" ]; then
> - line="[trusted=yes] ${line}"
> - fi
> - echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> - line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> - if [ -z "${BASE_REPO_KEY}" ]; then
> - line="[trusted=yes] ${line}"
> - fi
> - echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> -
> - mkdir -p ${ROOTFSDIR}/base-apt
> - mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
> - else
> - install -v -m644 "${APTSRCS}" \
> - "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> - fi
> - install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
> - rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> - rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
> - find ${APT_KEYS_DIR}/ -type f | while read keyfile
> - do
> - MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
> - echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
> - export GNUPGHOME="${MY_GPGHOME}"
> - APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
> -
> - kfn="$(basename $keyfile)"
> - cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
> - chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
> - --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
> - rm "${ROOTFSDIR}/tmp/$kfn"
> -
> - echo "Removing ${MY_GPGHOME}"
> - rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
> - done
> -
> - # Set locale
> - install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
> -
> - sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> - chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> -
> # setup chroot
> install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>
> - # update APT
> - mount -o bind,private /dev ${ROOTFSDIR}/dev
> - mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
> - mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
> - mount -t proc none ${ROOTFSDIR}/proc
> - mount --rbind /sys ${ROOTFSDIR}/sys
> - mount --make-rslave ${ROOTFSDIR}/sys
> -
> - export DEBIAN_FRONTEND=noninteractive
> -
> - if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
> - chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
> - fi
> -
> - if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> - chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
> - fi
> -
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
> - -o APT::Update::Error-Mode=any
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> - -o Debug::pkgProblemResolver=yes
> -
> - umount -l "${ROOTFSDIR}/dev/shm"
> - umount -l "${ROOTFSDIR}/dev/pts"
> - umount -l "${ROOTFSDIR}/dev"
> - umount -l "${ROOTFSDIR}/proc"
> - umount -l "${ROOTFSDIR}/sys"
> - umount -l "${ROOTFSDIR}/base-apt" || true
> -
> # Finalize debootstrap by setting the link in deploy
> ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> EOSUDO
> - deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> -
> - # Cleanup apt cache
> - sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
> + if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
> + deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> + sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
> + fi
> }
>
> addtask bootstrap before do_build after do_generate_keyrings
Jan
--
Siemens AG, Technology
Linux Expert Center
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 1/6] meta: Switch " Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
2023-09-27 10:07 ` Jan Kiszka
2023-09-27 4:33 ` [PATCH v4 4/6] user_manual.md: Replace debootstrap with mmdebstrap Uladzimir Bely
` (2 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
From: Anton Mikanovich <amikan@ilbers.de>
Do not unpack tar with rootfs content inside bootstrap recipe.
This also requires moving chroot prepare step to the later unpack step.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
meta/classes/rootfs.bbclass | 11 +++++----
.../isar-bootstrap/isar-bootstrap.inc | 23 ++++---------------
2 files changed, 12 insertions(+), 22 deletions(-)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 7dec7b36..09d40563 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -73,12 +73,15 @@ rootfs_do_qemu() {
fi
}
-BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
-BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
+BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
rootfs_prepare[weight] = "25"
rootfs_prepare(){
- sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+ sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
+
+ # setup chroot
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
}
ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
@@ -222,7 +225,7 @@ cache_deb_src() {
# Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
# relies on that through out the build. Copy that state information instead of apt-get update
# which generates a new state from upstream.
- sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
+ sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index d8ac9d00..cb1fc91a 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -16,7 +16,6 @@ PV = "1.0"
BOOTSTRAP_FOR_HOST ?= "0"
DEBOOTSTRAP = "mmdebstrap"
-ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
@@ -287,9 +286,6 @@ do_bootstrap() {
fi
E="${@ isar_export_proxies(d)}"
- sudo rm -rf --one-file-system "${ROOTFSDIR}"
- mkdir -p "${ROOTFSDIR}"
-
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
@@ -394,18 +390,9 @@ do_bootstrap() {
"${WORKDIR}/rootfs.tar.zst" \
"$bootstrap_list"
- sudo -E -s <<'EOSUDO'
- set -e
-
- tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
-
- # setup chroot
- install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+ # Finalize debootstrap by setting the link in deploy
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
- # Finalize debootstrap by setting the link in deploy
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
-EOSUDO
if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
@@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
bootstrap_sstate_prepare() {
# this runs in SSTATE_BUILDDIR, which will be deleted automatically
- sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
+ sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
sudo chown $(id -u):$(id -g) bootstrap.tar.zst
}
bootstrap_sstate_finalize() {
# this runs in SSTATE_INSTDIR
if [ -f bootstrap.tar.zst ]; then
- mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
- sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
+ mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
+ sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
"${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
fi
}
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory
2023-09-27 4:33 ` [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory Uladzimir Bely
@ 2023-09-27 10:07 ` Jan Kiszka
2023-10-17 14:05 ` Anton Mikanovich
0 siblings, 1 reply; 11+ messages in thread
From: Jan Kiszka @ 2023-09-27 10:07 UTC (permalink / raw)
To: Uladzimir Bely, isar-users
On 27.09.23 06:33, Uladzimir Bely wrote:
> From: Anton Mikanovich <amikan@ilbers.de>
>
> Do not unpack tar with rootfs content inside bootstrap recipe.
Motivating this change here would be nice.
> This also requires moving chroot prepare step to the later unpack step.
>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> meta/classes/rootfs.bbclass | 11 +++++----
> .../isar-bootstrap/isar-bootstrap.inc | 23 ++++---------------
> 2 files changed, 12 insertions(+), 22 deletions(-)
>
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 7dec7b36..09d40563 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -73,12 +73,15 @@ rootfs_do_qemu() {
> fi
> }
>
> -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
> -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
> +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
zst is really slow on packing but fast on unpacking IIRC. Do we actually
benefit from that (significantly more unpack than pack)?
Or was it like that before already?
>
> rootfs_prepare[weight] = "25"
> rootfs_prepare(){
> - sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> + sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
> +
> + # setup chroot
> + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> }
>
> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
> @@ -222,7 +225,7 @@ cache_deb_src() {
> # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
> # relies on that through out the build. Copy that state information instead of apt-get update
> # which generates a new state from upstream.
> - sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
> + sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
>
> deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
> debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index d8ac9d00..cb1fc91a 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -16,7 +16,6 @@ PV = "1.0"
> BOOTSTRAP_FOR_HOST ?= "0"
>
> DEBOOTSTRAP = "mmdebstrap"
> -ROOTFSDIR = "${WORKDIR}/rootfs"
> APTPREFS = "${WORKDIR}/apt-preferences"
> APTSRCS = "${WORKDIR}/apt-sources"
> APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
> @@ -287,9 +286,6 @@ do_bootstrap() {
> fi
> E="${@ isar_export_proxies(d)}"
>
> - sudo rm -rf --one-file-system "${ROOTFSDIR}"
> - mkdir -p "${ROOTFSDIR}"
> -
> if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
> bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
> @@ -394,18 +390,9 @@ do_bootstrap() {
> "${WORKDIR}/rootfs.tar.zst" \
> "$bootstrap_list"
>
> - sudo -E -s <<'EOSUDO'
> - set -e
> -
> - tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
> -
> - # setup chroot
> - install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> - "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> + # Finalize debootstrap by setting the link in deploy
> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>
> - # Finalize debootstrap by setting the link in deploy
> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> -EOSUDO
> if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
> deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
> @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>
> bootstrap_sstate_prepare() {
> # this runs in SSTATE_BUILDDIR, which will be deleted automatically
> - sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
> + sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
> sudo chown $(id -u):$(id -g) bootstrap.tar.zst
> }
>
> bootstrap_sstate_finalize() {
> # this runs in SSTATE_INSTDIR
> if [ -f bootstrap.tar.zst ]; then
> - mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> - sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> + mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
> "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
> fi
> }
Jan
--
Siemens AG, Technology
Linux Expert Center
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory
2023-09-27 10:07 ` Jan Kiszka
@ 2023-10-17 14:05 ` Anton Mikanovich
0 siblings, 0 replies; 11+ messages in thread
From: Anton Mikanovich @ 2023-10-17 14:05 UTC (permalink / raw)
To: Jan Kiszka, Uladzimir Bely, isar-users
27/09/2023 13:07, 'Jan Kiszka' via isar-users wrote:
> On 27.09.23 06:33, Uladzimir Bely wrote:
>> From: Anton Mikanovich <amikan@ilbers.de>
>>
>> Do not unpack tar with rootfs content inside bootstrap recipe.
> Motivating this change here would be nice.
>
>> This also requires moving chroot prepare step to the later unpack step.
>>
>> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
>> ---
>> meta/classes/rootfs.bbclass | 11 +++++----
>> .../isar-bootstrap/isar-bootstrap.inc | 23 ++++---------------
>> 2 files changed, 12 insertions(+), 22 deletions(-)
>>
>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>> index 7dec7b36..09d40563 100644
>> --- a/meta/classes/rootfs.bbclass
>> +++ b/meta/classes/rootfs.bbclass
>> @@ -73,12 +73,15 @@ rootfs_do_qemu() {
>> fi
>> }
>>
>> -BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}"
>> -BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-host_${DISTRO}-${DISTRO_ARCH}.tar.zst"
>> +BOOTSTRAP_SRC:${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}.tar.zst"
> zst is really slow on packing but fast on unpacking IIRC. Do we actually
> benefit from that (significantly more unpack than pack)?
>
> Or was it like that before already?
Zstd was chosen because it was already used for sstate.
Packing counts will always be <= unpacks, so it should be ok.
Other feedback will be addressed in v5.
>> rootfs_prepare[weight] = "25"
>> rootfs_prepare(){
>> - sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>> + sudo tar -xf "${BOOTSTRAP_SRC}" -C "${ROOTFSDIR}" --exclude="./dev/console"
>> +
>> + # setup chroot
>> + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>> }
>>
>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>> @@ -222,7 +225,7 @@ cache_deb_src() {
>> # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
>> # relies on that through out the build. Copy that state information instead of apt-get update
>> # which generates a new state from upstream.
>> - sudo cp -Trpn --reflink=auto "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
>> + sudo tar -xf "${BOOTSTRAP_SRC}" ./var/lib/apt/lists --one-top-level="${ROOTFSDIR}"
>>
>> deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
>> debsrc_download ${ROOTFSDIR} ${ROOTFS_BASE_DISTRO}-${BASE_DISTRO_CODENAME}
>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> index d8ac9d00..cb1fc91a 100644
>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> @@ -16,7 +16,6 @@ PV = "1.0"
>> BOOTSTRAP_FOR_HOST ?= "0"
>>
>> DEBOOTSTRAP = "mmdebstrap"
>> -ROOTFSDIR = "${WORKDIR}/rootfs"
>> APTPREFS = "${WORKDIR}/apt-preferences"
>> APTSRCS = "${WORKDIR}/apt-sources"
>> APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
>> @@ -287,9 +286,6 @@ do_bootstrap() {
>> fi
>> E="${@ isar_export_proxies(d)}"
>>
>> - sudo rm -rf --one-file-system "${ROOTFSDIR}"
>> - mkdir -p "${ROOTFSDIR}"
>> -
>> if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>> base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
>> bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
>> @@ -394,18 +390,9 @@ do_bootstrap() {
>> "${WORKDIR}/rootfs.tar.zst" \
>> "$bootstrap_list"
>>
>> - sudo -E -s <<'EOSUDO'
>> - set -e
>> -
>> - tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
>> -
>> - # setup chroot
>> - install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
>> - "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>> + # Finalize debootstrap by setting the link in deploy
>> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>>
>> - # Finalize debootstrap by setting the link in deploy
>> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
>> -EOSUDO
>> if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
>> deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
>> sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
>> @@ -420,15 +407,15 @@ SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
>>
>> bootstrap_sstate_prepare() {
>> # this runs in SSTATE_BUILDDIR, which will be deleted automatically
>> - sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" ./bootstrap.tar.zst
>> + sudo cp -a "${WORKDIR}/rootfs.tar.zst" ./bootstrap.tar.zst
>> sudo chown $(id -u):$(id -g) bootstrap.tar.zst
>> }
>>
>> bootstrap_sstate_finalize() {
>> # this runs in SSTATE_INSTDIR
>> if [ -f bootstrap.tar.zst ]; then
>> - mv bootstrap.tar.zst "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
>> - sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
>> + mv bootstrap.tar.zst "${WORKDIR}/rootfs.tar.zst"
>> + sudo ln -Tfsr "${WORKDIR}/rootfs.tar.zst" \
>> "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>> fi
>> }
> Jan
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v4 4/6] user_manual.md: Replace debootstrap with mmdebstrap
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
` (2 preceding siblings ...)
2023-09-27 4:33 ` [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 5/6] user_manual.md: Update minimal host requirements Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 6/6] isar-bootstrap: Remove unused code Uladzimir Bely
5 siblings, 0 replies; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
From: Anton Mikanovich <amikan@ilbers.de>
Update documentation in user manual.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
doc/user_manual.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index be47fdcc..dc7ed4c8 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -67,7 +67,9 @@ Install the following packages:
apt install \
binfmt-support \
bzip2 \
- debootstrap \
+ mmdebstrap \
+ arch-test \
+ apt-utils \
dpkg-dev \
gettext-base \
git \
@@ -127,7 +129,7 @@ apt install qemu
### Setup Sudo
-Isar requires `sudo` rights without password to work with `chroot` and `debootstrap`. To add them, use the following steps:
+Isar requires `sudo` rights without password to work with `chroot`. To add them, use the following steps:
```
# visudo
```
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH v4 5/6] user_manual.md: Update minimal host requirements
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
` (3 preceding siblings ...)
2023-09-27 4:33 ` [PATCH v4 4/6] user_manual.md: Replace debootstrap with mmdebstrap Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 6/6] isar-bootstrap: Remove unused code Uladzimir Bely
5 siblings, 0 replies; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
From: Anton Mikanovich <amikan@ilbers.de>
Using mmdebstrap requires at least Debian Bullseye on host.
Update user manual with the new minimal host distro.
Also remove all Buster-related notes.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
doc/user_manual.md | 35 ++++++++++++-----------------------
1 file changed, 12 insertions(+), 23 deletions(-)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index dc7ed4c8..02cb471b 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -48,7 +48,7 @@ For demonstration purposes, Isar provides support for the following
configurations:
- QEMU ARM with Debian Buster
- - QEMU ARM64 with Debian Buster (for host >= buster)
+ - QEMU ARM64 with Debian Buster
- QEMU i386 with Debian Buster
- QEMU amd64 with Debian Buster
- Raspberry Pi various models with Raspberry OS Bullseye
@@ -60,7 +60,7 @@ The steps below describe how to build the images provided by default.
### Install Host Tools
-The supported host system is >= buster.
+The minimal supported host system is Debian Bullseye.
Install the following packages:
```
@@ -74,6 +74,7 @@ apt install \
gettext-base \
git \
python3 \
+ python3-distutils \
quilt \
qemu-user-static \
reprepro \
@@ -87,28 +88,19 @@ apt install \
zstd
```
-If your host is >= buster, also install the following package.
-```
-apt install python3-distutils
-```
-
-**NOTE:** sbuild version (<=0.78.1) packaged in Debian Buster doesn't support
-`$apt_keep_downloaded_packages` option which is required in Isar for
-populating `${DL_DIR}/deb`. So, host `sbuild` in this case should be manually
-upgraded to >=0.81.2 version from Debian Bullseye.
-
Next, the user who should run Isar needs to be added to `sbuild` group.
```
sudo gpasswd -a <username> sbuild
```
If you want to generate containerized SDKs, also install the following
-packages: `umoci` and `skopeo`.
-Umoci is provided by Debian Buster and can be installed with
-`apt install umoci`, Skopeo is provided by Debian Bullseye/Unstable and has to
-be installed either manually downloading the DEB and installing it (no other
-packages required) or with `apt install -t bullseye skopeo` (if
-unstable/bullseye included in `/etc/apt/sources.list[.d]`).
+packages:
+
+```
+apt install \
+ umoci \
+ skopeo
+```
Notes:
@@ -234,9 +226,6 @@ Variables may be used in `.wks.in` files; Isar will expand them and generate a r
In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
-Note that the `ovmf` package in Debian Buster contains a pre-compiled firmware, but doesn't seem to be recent
-enough to allow images to be testable under `qemu`.
-
```
# AMD64 image, EFI
qemu-system-x86_64 -m 256M -nographic -bios edk2/Build/OvmfX64/RELEASE_*/FV/OVMF.fd -hda tmp/deploy/images/qemuamd64/isar-image-base-debian-buster-qemuamd64.wic
@@ -963,8 +952,8 @@ put into Isar apt.
Debian cross-compilation works out of the box. Currently the following build configurations are supported in Isar:
- buster armhf
- - buster arm64 (for host >= buster)
- - buster mipsel (for host >= buster)
+ - buster arm64
+ - buster mipsel
- bullseye armhf
- bullseye arm64
- bullseye mipsel
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH v4 6/6] isar-bootstrap: Remove unused code
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
` (4 preceding siblings ...)
2023-09-27 4:33 ` [PATCH v4 5/6] user_manual.md: Update minimal host requirements Uladzimir Bely
@ 2023-09-27 4:33 ` Uladzimir Bely
5 siblings, 0 replies; 11+ messages in thread
From: Uladzimir Bely @ 2023-09-27 4:33 UTC (permalink / raw)
To: isar-users
From: Anton Mikanovich <amikan@ilbers.de>
Some external logic like installing gpg and ca-certificates is now
doing by mmdebstrap inself. So remove duplications.
Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
.../isar-bootstrap/isar-bootstrap.inc | 26 -------------------
1 file changed, 26 deletions(-)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cb1fc91a..885428f5 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -23,8 +23,6 @@ DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
-DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-certificates"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
@@ -176,30 +174,6 @@ def get_distro_primary_source_entry(d):
return source[2:]
bb.fatal('Invalid apt sources list')
-def get_distro_have_https_source(d):
- return any(source[2].startswith("https://") for source in generate_distro_sources(d))
-
-def get_distro_needs_https_support(d):
- if get_distro_have_https_source(d):
- return "https-support"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
-
-def get_distro_needs_gpg_support(d):
- if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
- d.getVar("THIRD_PARTY_APT_KEYS") or \
- d.getVar("BASE_REPO_KEY"):
- return "gnupg"
- else:
- return ""
-
-OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
-
-def get_distro_source(d):
- return get_distro_primary_source_entry(d)[0]
-
def get_distro_suite(d):
return get_distro_primary_source_entry(d)[1]
--
2.20.1
^ permalink raw reply [flat|nested] 11+ messages in thread