From: Jan Kiszka <jan.kiszka@siemens.com>
To: Uladzimir Bely <ubely@ilbers.de>, isar-users@googlegroups.com
Subject: Re: [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks
Date: Wed, 27 Sep 2023 12:05:23 +0200 [thread overview]
Message-ID: <efea3564-1b90-4383-a8c3-878b5cbf3526@siemens.com> (raw)
In-Reply-To: <20230927043320.6264-3-ubely@ilbers.de>
On 27.09.23 06:33, Uladzimir Bely wrote:
> From: Anton Mikanovich <amikan@ilbers.de>
>
> Move all bootstrap rootfs prepare logic into mmdebstrap hooks.
>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
> .../isar-bootstrap/isar-bootstrap.inc | 196 +++++++++---------
> 1 file changed, 102 insertions(+), 94 deletions(-)
>
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 637018c6..d8ac9d00 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -211,24 +211,21 @@ def get_distro_components_argument(d):
> else:
> return ""
>
> -APT_KEYS_DIR = "${WORKDIR}/aptkeys"
> DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
>
> -do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
> -do_generate_keyrings[dirs] = "${DL_DIR}"
> +do_generate_keyrings[cleandirs] = "${WORKDIR}/trusted.gpg.d"
> +do_generate_keyrings[dirs] = "${DEBDIR}"
> do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS THIRD_PARTY_APT_KEYS"
> do_generate_keyrings[network] = "${TASK_USE_SUDO}"
> do_generate_keyrings() {
> if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
> - chmod 777 "${APT_KEYS_DIR}"
> for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
> - cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> + sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
Isn't apt-key deprecated? Why re-introducing it then?
> done
> fi
> if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ]; then
> for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
> sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add $keyfile
> - cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> done
> fi
> }
> @@ -269,12 +266,14 @@ do_bootstrap[vardeps] += " \
> ISAR_ENABLE_COMPAT_ARCH \
> ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> "
> -do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR} ${WORKDIR}/trusted.gpg.d ${WORKDIR}/sources.list.d"
> do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
>
> inherit compat
>
> +DEB_DL_LOCK ?= "${DEBDIR}/${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}.lock"
> +
> do_bootstrap() {
> if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
> @@ -284,124 +283,133 @@ do_bootstrap() {
> debootstrap_args="--verbose --variant=minbase --include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
> if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> debootstrap_args="$debootstrap_args --keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> + cp "${DISTRO_BOOTSTRAP_KEYRING}" "${WORKDIR}/trusted.gpg.d/"
> fi
> E="${@ isar_export_proxies(d)}"
> - export BOOTSTRAP_FOR_HOST
>
> - deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> sudo rm -rf --one-file-system "${ROOTFSDIR}"
> mkdir -p "${ROOTFSDIR}"
>
> + if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> + base_apt_tmp="$(mktemp -d /tmp/isar-base-aptXXXXXXXXXX)"
> + bootstrap_list="${WORKDIR}/sources.list.d/base-apt.list"
> + line="copy://$base_apt_tmp/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb ${line}" > "${WORKDIR}/sources.list.d/base-apt.list"
> + line="copy://$base_apt_tmp/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> + if [ -z "${BASE_REPO_KEY}" ]; then
> + line="[trusted=yes] ${line}"
> + fi
> + echo "deb-src ${line}" >> "${WORKDIR}/sources.list.d/base-apt.list"
> +
> + # no need to sync /var/cache/apt/archives if base-apt used
> + syncin='echo skip sync-in'
> + syncout='echo skip sync-out'
> + extra_setup="mount --bind '${REPO_BASE_DIR}' $base_apt_tmp"
> + extra_extract="$syncout"
> + # save mmdebstrap tempdir for cleanup
> + extra_essential="mkdir -p \$1/$base_apt_tmp && \
> + echo \$1 > ${WORKDIR}/mmtmpdir && \
> + mount --bind '${REPO_BASE_DIR}' \$1/$base_apt_tmp"
> + # replace base-apt mount in tmp with /base-apt mount
> + extra_customize="sed -i \"s|copy://$base_apt_tmp|file:///base-apt|g\" \
> + \$1/etc/apt/sources.list.d/*.list && \
> + mkdir -p \$1/base-apt && \
> + mount --bind '${REPO_BASE_DIR}' \$1/base-apt && \
> + chroot \$1 apt-get update -y \
> + -o APT::Update::Error-Mode=any && \
> + umount \$1/base-apt && \
> + umount \$1/$base_apt_tmp && rm ${WORKDIR}/mmtmpdir && \
> + umount $base_apt_tmp && rm -rf --one-file-system $base_apt_tmp"
> + else
> + deb_dl_dir_import "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> +
> + bootstrap_list="${WORKDIR}/sources.list.d/bootstrap.list"
> + install -v -m644 "${APTSRCS}" \
> + "${WORKDIR}/sources.list.d/bootstrap.list"
> +
> + syncin='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
> + "${WORKDIR}/dl_dir/var/cache/apt/archives/"*.deb \
> + "$1/var/cache/apt/archives/" || true'
> + syncout='flock -s ${DEB_DL_LOCK} cp -n --no-preserve=owner \
> + "$1/var/cache/apt/archives/"*.deb \
> + "${WORKDIR}/dl_dir/var/cache/apt/archives/"'
> + extra_setup="$syncin"
> + extra_extract="$syncout"
> + # prefetch apt debs because mmdebstrap will clean them on next stage
> + extra_essential='apt-get install apt -y -d \
> + -o Dir::State="$1/var/lib/apt" \
> + -o Dir::Etc="$1/etc/apt" \
> + -o Dir::Cache="$1/var/cache/apt" \
> + -o Apt::Architecture="${BOOTSTRAP_DISTRO_ARCH}"'
> + extra_essential="$extra_essential && $syncout"
> + extra_customize="$syncout"
> + fi
> +
> if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> fi
>
> arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> + if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> + arch_param="$arch_param,${COMPAT_DISTRO_ARCH}"
> + fi
> +
> + # Cleanup mounts if fails
> + trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> + trap '[ -r "${WORKDIR}/mmtmpdir" ] && tmpdir=$(cat "${WORKDIR}/mmtmpdir") \
> + && rm "${WORKDIR}/mmtmpdir"; \
> + [ -d "$tmpdir" ] && mountpoint -q $tmpdir/$base_apt_tmp \
> + && sudo umount $tmpdir/$base_apt_tmp; \
> + [ -d "$tmpdir" ] && mountpoint -q $tmpdir/base-apt \
> + && sudo umount $tmpdir/base-apt; \
> + [ -d "$tmpdir" ] && sudo rm -rf --one-file-system $tmpdir; \
> + [ -n "$base_apt_tmp" ] && mountpoint -q $base_apt_tmp \
> + && sudo umount $base_apt_tmp \
> + && rm -rf --one-file-system $base_apt_tmp' EXIT
>
> sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP} $debootstrap_args \
> $arch_param \
> --mode=unshare \
> + --setup-hook='mkdir -p "$1/var/cache/apt/archives/"' \
> + --setup-hook="$extra_setup" \
> + --setup-hook='upload "${APTPREFS}" /etc/apt/preferences.d/bootstrap' \
> + --setup-hook='upload "${APTSRCS_INIT}" /etc/apt/sources-list' \
> + --setup-hook='upload "${WORKDIR}/locale" /etc/locale' \
> + --setup-hook='mkdir -p "$1/etc/apt/trusted.gpg.d"' \
> + --setup-hook='sync-in "${WORKDIR}/trusted.gpg.d" /etc/apt/trusted.gpg.d' \
> + --setup-hook='install -v -m755 "${WORKDIR}/chroot-setup.sh" "$1/chroot-setup.sh"' \
> + --extract-hook="$extra_extract" \
> + --essential-hook="$extra_essential" \
> + --customize-hook="$extra_customize" \
> + --customize-hook='sed -i "/en_US.UTF-8 UTF-8/s/^#//g" "$1/etc/locale.gen"' \
> + --customize-hook='chroot "$1" /usr/sbin/locale-gen' \
> + --customize-hook='chroot "$1" /usr/bin/apt-get -y clean' \
> + --skip=cleanup/apt \
> + --skip=download/empty \
> ${@get_distro_components_argument(d)} \
> "${@get_distro_suite(d)}" \
> "${WORKDIR}/rootfs.tar.zst" \
> - "${@get_distro_source(d)}"
> + "$bootstrap_list"
>
> sudo -E -s <<'EOSUDO'
> set -e
>
> tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --exclude="./dev/console"
>
> - # Install apt config
> - mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> - install -v -m644 "${APTPREFS}" \
> - "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> - mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> - line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> - if [ -z "${BASE_REPO_KEY}" ]; then
> - line="[trusted=yes] ${line}"
> - fi
> - echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> - line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> - if [ -z "${BASE_REPO_KEY}" ]; then
> - line="[trusted=yes] ${line}"
> - fi
> - echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> -
> - mkdir -p ${ROOTFSDIR}/base-apt
> - mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
> - else
> - install -v -m644 "${APTSRCS}" \
> - "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> - fi
> - install -v -m644 "${APTSRCS_INIT}" "${ROOTFSDIR}/etc/apt/sources-list"
> - rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> - rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
> - find ${APT_KEYS_DIR}/ -type f | while read keyfile
> - do
> - MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d /tmp/gpghomeXXXXXXXXXX)"
> - echo "Created temporary directory ${MY_GPGHOME} for gpg-agent"
> - export GNUPGHOME="${MY_GPGHOME}"
> - APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
> -
> - kfn="$(basename $keyfile)"
> - cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
> - chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon -- /usr/bin/apt-key \
> - --keyring ${THIRD_PARTY_APT_KEYRING} ${APT_KEY_APPEND} add "/tmp/$kfn"
> - rm "${ROOTFSDIR}/tmp/$kfn"
> -
> - echo "Removing ${MY_GPGHOME}"
> - rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
> - done
> -
> - # Set locale
> - install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
> -
> - sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> - chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> -
> # setup chroot
> install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
>
> - # update APT
> - mount -o bind,private /dev ${ROOTFSDIR}/dev
> - mount --bind /dev/pts ${ROOTFSDIR}/dev/pts
> - mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
> - mount -t proc none ${ROOTFSDIR}/proc
> - mount --rbind /sys ${ROOTFSDIR}/sys
> - mount --make-rslave ${ROOTFSDIR}/sys
> -
> - export DEBIAN_FRONTEND=noninteractive
> -
> - if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
> - chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
> - fi
> -
> - if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> - chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${COMPAT_DISTRO_ARCH}
> - fi
> -
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
> - -o APT::Update::Error-Mode=any
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
> - chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> - -o Debug::pkgProblemResolver=yes
> -
> - umount -l "${ROOTFSDIR}/dev/shm"
> - umount -l "${ROOTFSDIR}/dev/pts"
> - umount -l "${ROOTFSDIR}/dev"
> - umount -l "${ROOTFSDIR}/proc"
> - umount -l "${ROOTFSDIR}/sys"
> - umount -l "${ROOTFSDIR}/base-apt" || true
> -
> # Finalize debootstrap by setting the link in deploy
> ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> EOSUDO
> - deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> -
> - # Cleanup apt cache
> - sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
> + if [ "${ISAR_USE_CACHED_BASE_REPO}" != "1" ]; then
> + deb_dl_dir_export "${WORKDIR}/dl_dir" "${BOOTSTRAP_BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> + sudo rm -rf --one-file-system "${WORKDIR}/dl_dir"
> + fi
> }
>
> addtask bootstrap before do_build after do_generate_keyrings
Jan
--
Siemens AG, Technology
Linux Expert Center
next prev parent reply other threads:[~2023-09-27 10:05 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-09-27 4:33 [PATCH v4 0/6] Migrate to mmdebstrap Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 1/6] meta: Switch " Uladzimir Bely
2023-09-27 10:02 ` Jan Kiszka
2023-09-27 4:33 ` [PATCH v4 2/6] isar-bootstrap: Move preparations to hooks Uladzimir Bely
2023-09-27 10:05 ` Jan Kiszka [this message]
2023-09-27 4:33 ` [PATCH v4 3/6] isar-bootstrap: Use tar output instead of directory Uladzimir Bely
2023-09-27 10:07 ` Jan Kiszka
2023-10-17 14:05 ` Anton Mikanovich
2023-09-27 4:33 ` [PATCH v4 4/6] user_manual.md: Replace debootstrap with mmdebstrap Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 5/6] user_manual.md: Update minimal host requirements Uladzimir Bely
2023-09-27 4:33 ` [PATCH v4 6/6] isar-bootstrap: Remove unused code Uladzimir Bely
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=efea3564-1b90-4383-a8c3-878b5cbf3526@siemens.com \
--to=jan.kiszka@siemens.com \
--cc=isar-users@googlegroups.com \
--cc=ubely@ilbers.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox