Re: [PATCH v11 2/8] meta: Add mmdebstrap recipe

["'MOESSBAUER, Felix' via isar-users" <isar-users@googlegroups.com>]

On Wed, 2024-11-06 at 10:21 +0200, Anton Mikanovich wrote:
> It can be used as debootstrap alternative for rootfs prepare.
> 
> Internally, it uses apt and allows to bootstrap the distro from
> multiple repositories.
> 
> chroot-setup.sh and locale are copied from debootstrap recipe.
> 
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
>  meta/classes/bootstrap.bbclass                |   1 +
>  .../isar-mmdebstrap/isar-mmdebstrap-host.bb   |  17 ++
>  .../isar-mmdebstrap/isar-mmdebstrap-target.bb |  12 +
>  .../isar-mmdebstrap/isar-mmdebstrap.inc       | 235
> ++++++++++++++++++
>  4 files changed, 265 insertions(+)
>  create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-host.bb
>  create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap-target.bb
>  create mode 100644 meta/recipes-core/isar-mmdebstrap/isar-
> mmdebstrap.inc
> 
> diff --git a/meta/classes/bootstrap.bbclass
> b/meta/classes/bootstrap.bbclass
> index 870465a9..f5b92808 100644
> --- a/meta/classes/bootstrap.bbclass
> +++ b/meta/classes/bootstrap.bbclass
> @@ -26,6 +26,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES ??= ""
>  DISTRO_VARS_PREFIX ?= "${@'HOST_' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else ''}"
>  BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else 'DISTRO')}"
>  BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else
> 'BASE_DISTRO')}"
> +BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if
> bb.utils.to_boolean(d.getVar('BOOTSTRAP_FOR_HOST')) else
> 'DISTRO_ARCH')}"
>  ISAR_APT_SNAPSHOT_DATE ?= "${@ get_isar_apt_snapshot_date(d)}"
>  
>  python () {
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> host.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
> new file mode 100644
> index 00000000..66c8d11e
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-host.bb
> @@ -0,0 +1,17 @@
> +# Minimal host Debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +Description = "Minimal host Debian root file system"
> +
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${HOST_DISTRO}-
> host_${DISTRO}-${DISTRO_ARCH}"
> +
> +BOOTSTRAP_FOR_HOST = "1"
> +
> +require isar-mmdebstrap.inc
> +
> +HOST_DISTRO_BOOTSTRAP_KEYS ?= ""
> +DISTRO_BOOTSTRAP_KEYS = "${HOST_DISTRO_BOOTSTRAP_KEYS}"
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> target.bb b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-
> target.bb
> new file mode 100644
> index 00000000..84a89ff1
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap-target.bb
> @@ -0,0 +1,12 @@
> +# Minimal target Debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +Description = "Minimal target Debian root file system"
> +
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_BOOTSTRAP}/${DISTRO}-
> ${DISTRO_ARCH}"
> +
> +require isar-mmdebstrap.inc
> diff --git a/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> new file mode 100644
> index 00000000..658d45be
> --- /dev/null
> +++ b/meta/recipes-core/isar-mmdebstrap/isar-mmdebstrap.inc
> @@ -0,0 +1,235 @@
> +# Minimal debian root file system
> +#
> +# This software is a part of Isar.
> +# Copyright (C) 2024 ilbers GmbH
> +#
> +# SPDX-License-Identifier: MIT
> +
> +inherit bootstrap
> +inherit compat
> +inherit deb-dl-dir
> +
> +FILESEXTRAPATHS:append = ":${LAYERDIR_core}/recipes-core/isar-
> bootstrap/files"
> +
> +ROOTFSDIR = "${WORKDIR}/rootfs"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,apt,usrmerge"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-
> certificates"
> +BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
> +
> +def get_distro_primary_source_entry(d):
> +    for source in generate_distro_sources(d):
> +        if source[0] == "deb":
> +            return source[2:]
> +    bb.fatal('Invalid apt sources list')
> +
> +def get_distro_have_https_source(d):
> +    return any(source[2].startswith("https://") for source in
> generate_distro_sources(d))
> +
> +def get_distro_needs_https_support(d):
> +    if get_distro_have_https_source(d):
> +        return "https-support"
> +    else:
> +        return ""
> +
> +OVERRIDES:append = ":${@get_distro_needs_https_support(d)}"
> +
> +def get_distro_needs_gpg_support(d):
> +    if d.getVar("DISTRO_BOOTSTRAP_KEYS") or \
> +       d.getVar("THIRD_PARTY_APT_KEYS") or \
> +       d.getVar("BASE_REPO_KEY"):
> +        return "gnupg"
> +    else:
> +        return ""
> +
> +OVERRIDES:append = ":${@get_distro_needs_gpg_support(d)}"
> +
> +APT_KEYS_DIR = "${WORKDIR}/aptkeys"
> +DISTRO_BOOTSTRAP_KEYRING = "${WORKDIR}/distro-keyring.gpg"
> +
> +do_generate_keyrings[cleandirs] = "${APT_KEYS_DIR}"
> +do_generate_keyrings[dirs] = "${DL_DIR}"
> +do_generate_keyrings[vardeps] += "DISTRO_BOOTSTRAP_KEYS
> THIRD_PARTY_APT_KEYS"
> +do_generate_keyrings[network] = "${TASK_USE_SUDO}"
> +do_generate_keyrings() {
> +    if [ -n "${@d.getVar("THIRD_PARTY_APT_KEYFILES") or ""}" ]; then
> +        chmod 777 "${APT_KEYS_DIR}"
> +        for keyfile in ${@d.getVar("THIRD_PARTY_APT_KEYFILES")}; do
> +           cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> +        done
> +    fi
> +    if [ -n "${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES") or ""}" ];
> then
> +        for keyfile in ${@d.getVar("DISTRO_BOOTSTRAP_KEYFILES")}; do
> +           sudo apt-key --keyring "${DISTRO_BOOTSTRAP_KEYRING}" add
> $keyfile
> +           cp "$keyfile" "${APT_KEYS_DIR}"/"$(basename "$keyfile")"
> +        done
> +    fi
> +}
> +addtask generate_keyrings before do_build after do_unpack
> +
> +do_bootstrap[vardeps] += " \
> +    DISTRO_APT_PREMIRRORS \
> +    ISAR_ENABLE_COMPAT_ARCH \
> +    ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
> +    "
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
> +do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
> +do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
> +
> +do_bootstrap() {
> +    if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> +        if [ -z "${COMPAT_DISTRO_ARCH}" ]; then
> +            bbfatal "${DISTRO_ARCH} does not have a compat arch"
> +        fi
> +    fi
> +    bootstrap_args="--verbose --variant=minbase --
> include=${DISTRO_BOOTSTRAP_BASE_PACKAGES}"
> +    if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
> +        bootstrap_args="$bootstrap_args --
> keyring=${DISTRO_BOOTSTRAP_KEYRING}"
> +    fi
> +    E="${@ isar_export_proxies(d)}"
> +    export BOOTSTRAP_FOR_HOST
> +
> +    deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> +    sudo rm -rf --one-file-system "${ROOTFSDIR}"
> +    mkdir -p "${ROOTFSDIR}"
> +
> +    arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> +
> +    sudo TMPDIR="${BOOTSTRAP_TMPDIR}" mmdebstrap $bootstrap_args \

Hi, why does this need to run as root? mmdebstrap with the unshare
backend can also run unprivileged.

> +                   $arch_param \
> +                   --mode=unshare \
> +                   ${@get_distro_components_argument(d)} \
> +                   "${@get_distro_suite(d)}" \
> +                   "${WORKDIR}/rootfs.tar.zst" \
> +                   "${@get_distro_source(d)}"

The apt options are missing. This is especially relevant when running
against the snapshot mirrors. In debootstrap this was not possible, but
as mmdepstrap uses the host apt, options can be passed (with --aptopt
<file>).

Felix

> +
> +    sudo -E -s <<'EOSUDO'
> +        set -e
> +
> +        tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --
> exclude="./dev/console"
> +
> +        # Install apt config
> +        mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> +        install -v -m644 "${APTPREFS}" \
> +                        
> "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> +        mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> +        if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> +            line="file:///base-
> apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> +            if [ -z "${BASE_REPO_KEY}" ]; then
> +                line="[trusted=yes] ${line}"
> +            fi
> +            echo "deb ${line}" > 
> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> +            line="file:///base-
> apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> +            if [ -z "${BASE_REPO_KEY}" ]; then
> +                line="[trusted=yes] ${line}"
> +            fi
> +            echo "deb-src ${line}" >> 
> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> +
> +            mkdir -p ${ROOTFSDIR}/base-apt
> +            mount -o bind,private ${REPO_BASE_DIR}
> ${ROOTFSDIR}/base-apt
> +        else
> +            install -v -m644 "${APTSRCS}" \
> +                            
> "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> +        fi
> +        install -v -m644 "${APTSRCS_INIT}"
> "${ROOTFSDIR}/etc/apt/sources-list"
> +        rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> +        rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
> +        find ${APT_KEYS_DIR}/ -type f | while read keyfile
> +        do
> +            MY_GPGHOME="$(chroot "${ROOTFSDIR}" mktemp -d
> /tmp/gpghomeXXXXXXXXXX)"
> +            echo "Created temporary directory ${MY_GPGHOME} for gpg-
> agent"
> +            export GNUPGHOME="${MY_GPGHOME}"
> +            APT_KEY_APPEND="--homedir ${MY_GPGHOME}"
> +
> +            kfn="$(basename $keyfile)"
> +            cp $keyfile "${ROOTFSDIR}/tmp/$kfn"
> +            chroot "${ROOTFSDIR}" /usr/bin/gpg-agent --daemon --
> /usr/bin/apt-key \
> +                --keyring ${THIRD_PARTY_APT_KEYRING}
> ${APT_KEY_APPEND} add "/tmp/$kfn"
> +            rm "${ROOTFSDIR}/tmp/$kfn"
> +
> +            echo "Removing ${MY_GPGHOME}"
> +            rm -rf "${ROOTFSDIR}${MY_GPGHOME}"
> +        done
> +
> +        # Set locale
> +        install -v -m644 "${WORKDIR}/locale"
> "${ROOTFSDIR}/etc/locale"
> +
> +        sed -i '/en_US.UTF-8 UTF-8/s/^#//g'
> "${ROOTFSDIR}/etc/locale.gen"
> +        chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> +
> +        # update APT
> +        mount -o bind,private /dev ${ROOTFSDIR}/dev
> +        mount -o bind,private /dev/pts ${ROOTFSDIR}/dev/pts
> +        mount -t tmpfs none "${ROOTFSDIR}/dev/shm"
> +        mount -t proc none ${ROOTFSDIR}/proc
> +        mount -o bind,private /sys ${ROOTFSDIR}/sys
> +        mount --make-rslave ${ROOTFSDIR}/sys
> +
> +        export DEBIAN_FRONTEND=noninteractive
> +
> +        if [ "${BOOTSTRAP_FOR_HOST}" = "1" ]; then
> +            chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
> ${DISTRO_ARCH}
> +        fi
> +
> +        if [ "${ISAR_ENABLE_COMPAT_ARCH}" = "1" ]; then
> +            chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture
> ${COMPAT_DISTRO_ARCH}
> +        fi
> +
> +        chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y \
> +                                -o APT::Update::Error-Mode=any
> +
> +        chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y dpkg
> +
> +        # setup chroot
> +        install -v -m755 "${WORKDIR}/chroot-setup.sh"
> "${ROOTFSDIR}/chroot-setup.sh"
> +        "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> +
> +        chroot "${ROOTFSDIR}" /usr/bin/apt-get install -y -f
> +        chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> +                                -o Debug::pkgProblemResolver=yes
> +
> +        umount "${ROOTFSDIR}/dev/shm"
> +        umount "${ROOTFSDIR}/dev/pts"
> +        umount "${ROOTFSDIR}/dev"
> +        umount "${ROOTFSDIR}/proc"
> +        umount "${ROOTFSDIR}/sys"
> +        if mountpoint -q "${ROOTFSDIR}/base-apt"; then
> +            umount "${ROOTFSDIR}/base-apt"
> +        fi
> +
> +        # Finalize bootstrap by setting the link in deploy
> +        ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> +EOSUDO
> +    deb_dl_dir_export "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> +
> +    # Cleanup apt cache
> +    sudo -Es chroot "${ROOTFSDIR}" /usr/bin/apt-get -y clean
> +}
> +addtask bootstrap before do_build after do_generate_keyrings
> +
> +SSTATETASKS += "do_bootstrap"
> +SSTATECREATEFUNCS += "bootstrap_sstate_prepare"
> +SSTATEPOSTINSTFUNCS += "bootstrap_sstate_finalize"
> +
> +bootstrap_sstate_prepare() {
> +    # this runs in SSTATE_BUILDDIR, which will be deleted
> automatically
> +    sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> ./bootstrap.tar.zst
> +    sudo chown $(id -u):$(id -g) bootstrap.tar.zst
> +}
> +
> +bootstrap_sstate_finalize() {
> +    # this runs in SSTATE_INSTDIR
> +    # we should restore symlinks after using tar
> +    if [ -f bootstrap.tar.zst ]; then
> +        mv bootstrap.tar.zst "$(dirname
> "${ROOTFSDIR}")/rootfs.tar.zst"
> +        sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> +                      "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
> +    fi
> +}
> +
> +python do_bootstrap_setscene() {
> +    sstate_setscene(d)
> +}
> +
> +addtask do_bootstrap_setscene
> +do_bootstrap_setscene[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> -- 
> 2.34.1
> 

-- 
Siemens AG, Technology
Linux Expert Center


-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/1853b41ea5fb9b099b1147bb5c65e9a0188d1985.camel%40siemens.com.