Re: [PATCH v2 1/6] meta: Switch to mmdebstrap

["MOESSBAUER, Felix" <felix.moessbauer@siemens.com>]

On Thu, 2023-05-18 at 12:47 +0300, Anton Mikanovich wrote:
> From: Uladzimir Bely <ubely@ilbers.de>
> 
> Instead of debootstrap, use mmdebstrap alternative.
> 
> Internally, it uses apt and allows to debootstrap the distro from
> multiple repositories.

Hi,

does this solve the issue that we cannot bootstrap ubuntu > focal on a
debian < bookworm host? This is currently not supported, as some debs
use zstd compression which is only supported from dpkg >= 1.21 on.

I hope mmdebstrap can work around this by not relying on the host dpkg,
as otherwise very soon bootstrapping from non bookworm hosts will not
be possible anymore. However, looking at the dependencies of mmdebstrap
I'm not too optimistic about that.

PS: This is also another reason to use bookworm as a base for the kas-
container (and new ISAR CI).

Felix

> 
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
> ---
>  meta-isar/conf/distro/ubuntu-focal.conf       |  4 --
>  .../isar-bootstrap/isar-bootstrap.inc         | 57 ++++++++++-------
> --
>  2 files changed, 30 insertions(+), 31 deletions(-)
> 
> diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-
> isar/conf/distro/ubuntu-focal.conf
> index 0cb6958d..be8c951c 100644
> --- a/meta-isar/conf/distro/ubuntu-focal.conf
> +++ b/meta-isar/conf/distro/ubuntu-focal.conf
> @@ -17,10 +17,6 @@ BOOTSTRAP_KEY =
> "file://${LAYERDIR_isar}/conf/distro/ubuntu.public.key;sha256sum
>  DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
>  HOST_DISTRO_BOOTSTRAP_KEYS += "${BOOTSTRAP_KEY}"
>  
> -
> -# that is what debootstrap_1.0.118ubuntu1 does anyways
> -DISTRO_DEBOOTSTRAP_SCRIPT = "/usr/share/debootstrap/scripts/gutsy"
> -
>  DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
>  
>  DISTRO_KERNELS ?= " \
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index b94ae0bd..f89c284e 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -15,7 +15,7 @@ PV = "1.0"
>  
>  BOOTSTRAP_FOR_HOST ?= "0"
>  
> -DEBOOTSTRAP ?= "qemu-debootstrap"
> +DEBOOTSTRAP = "mmdebstrap"
>  ROOTFSDIR = "${WORKDIR}/rootfs"
>  APTPREFS = "${WORKDIR}/apt-preferences"
>  APTSRCS = "${WORKDIR}/apt-sources"
> @@ -23,12 +23,14 @@ APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
>  DISTRO_BOOTSTRAP_KEYFILES = ""
>  THIRD_PARTY_APT_KEYFILES = ""
>  DEPLOY_ISAR_BOOTSTRAP ?= ""
> -DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
> +DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales,usrmerge"
>  DISTRO_BOOTSTRAP_BASE_PACKAGES:append:gnupg = ",gnupg"
>  DISTRO_BOOTSTRAP_BASE_PACKAGES:append:https-support = ",ca-
> certificates"
>  DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST')
> == '1' else ''}"
>  BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
>  BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
> +BOOTSTRAP_DISTRO_ARCH = "${@d.getVar('HOST_ARCH' if
> d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO_ARCH')}"
> +BOOTSTRAP_TMPDIR = "${WORKDIR}/tempdir"
>  FILESEXTRAPATHS:append = ":${BBPATH}"
>  
>  inherit deb-dl-dir
> @@ -270,7 +272,7 @@ do_bootstrap[vardeps] += " \
>      ISAR_ENABLE_COMPAT_ARCH \
>      ${DISTRO_VARS_PREFIX}DISTRO_APT_SOURCES \
>      "
> -do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP} ${BOOTSTRAP_TMPDIR}"
>  do_bootstrap[depends] = "base-apt:do_cache isar-apt:do_cache_config"
>  do_bootstrap[network] = "${TASK_USE_NETWORK_AND_SUDO}"
>  
> @@ -286,27 +288,31 @@ do_bootstrap() {
>      if [ -f "${DISTRO_BOOTSTRAP_KEYRING}" ]; then
>          debootstrap_args="$debootstrap_args --
> keyring=${DISTRO_BOOTSTRAP_KEYRING}"
>      fi
> -    if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" -a -z
> "${BASE_REPO_KEY}" ]; then
> -        debootstrap_args="$debootstrap_args --no-check-gpg"
> -    fi
>      E="${@ isar_export_proxies(d)}"
> -    export BOOTSTRAP_FOR_HOST debootstrap_args E
> +    export BOOTSTRAP_FOR_HOST
>  
> -    sudo rm -rf --one-file-system "${ROOTFSDIR}"
>      deb_dl_dir_import "${ROOTFSDIR}" "${BOOTSTRAP_BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}"
> +    sudo rm -rf --one-file-system "${ROOTFSDIR}"
> +    mkdir -p "${ROOTFSDIR}"
> +
> +    if [ ! -z "${SOURCE_DATE_EPOCH}" ]; then
> +        export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}"
> +    fi
> +
> +    arch_param="--arch=${BOOTSTRAP_DISTRO_ARCH},${DISTRO_ARCH}"
> +
> +    sudo TMPDIR="${BOOTSTRAP_TMPDIR}" ${DEBOOTSTRAP}
> $debootstrap_args \
> +                   $arch_param \
> +                   --mode=unshare \
> +                   ${@get_distro_components_argument(d)} \
> +                   "${@get_distro_suite(d)}" \
> +                   "${WORKDIR}/rootfs.tar.zst" \
> +                   "${@get_distro_source(d)}"
>  
>      sudo -E -s <<'EOSUDO'
>          set -e
> -        if [ "${BOOTSTRAP_FOR_HOST}" = "0" ]; then
> -            arch_param="--arch=${DISTRO_ARCH}"
> -        fi
> -        ${DEBOOTSTRAP} $debootstrap_args \
> -                       $arch_param \
> -                       ${@get_distro_components_argument(d)} \
> -                       "${@get_distro_suite(d)}" \
> -                       "${ROOTFSDIR}" \
> -                       "${@get_distro_source(d)}" \
> -                       ${DISTRO_DEBOOTSTRAP_SCRIPT}
> +
> +        tar -xf "${WORKDIR}/rootfs.tar.zst" -C "${ROOTFSDIR}" --
> exclude="./dev/console"
>  
>          # Install apt config
>          mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> @@ -409,19 +415,16 @@ SSTATEPOSTINSTFUNCS +=
> "bootstrap_sstate_finalize"
>  
>  bootstrap_sstate_prepare() {
>      # this runs in SSTATE_BUILDDIR, which will be deleted
> automatically
> -    lopts="--one-file-system --exclude=var/cache/apt/archives"
> -    sudo tar -C $(dirname "${ROOTFSDIR}") -cpSf bootstrap.tar $lopts
> $(basename "${ROOTFSDIR}")
> -    sudo chown $(id -u):$(id -g) bootstrap.tar
> +    sudo cp -a "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst"
> ./bootstrap.tar.zst
> +    sudo chown $(id -u):$(id -g) bootstrap.tar.zst
>  }
>  
>  bootstrap_sstate_finalize() {
>      # this runs in SSTATE_INSTDIR
> -    # - after building the bootstrap, the tar won't be there, but we
> also don't need to unpack
> -    # - after restoring from cache, there will be a tar which we
> unpack and then delete
> -    if [ -f bootstrap.tar ]; then
> -        sudo tar -C $(dirname "${ROOTFSDIR}") -xpf bootstrap.tar
> -        sudo ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> -        rm bootstrap.tar
> +    if [ -f bootstrap.tar.zst ]; then
> +        mv bootstrap.tar.zst "$(dirname
> "${ROOTFSDIR}")/rootfs.tar.zst"
> +        sudo ln -Tfsr "$(dirname "${ROOTFSDIR}")/rootfs.tar.zst" \
> +                      "${DEPLOY_ISAR_BOOTSTRAP}.tar.zst"
>      fi
>  }
>  
> -- 
> 2.34.1
>