Re: [PATCH v2 1/5] implement isar-bootstrap using debootstrap

[Jan Kiszka <jan.kiszka@siemens.com>]

On 2018-02-22 14:18, [ext] claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
> 
> Since multistrap is deprecated for some years, it is required to change
> to debootstrap.
> 
> This patch introduces the 'isar-bootstrap' recipe that implement the
> creation of a minimal base root file system using debootstrap.
> 
> Signed-off-by: Claudius Heine <ch@denx.de>
> ---
>  meta-isar/conf/distro/debian-jessie.conf           |  14 ++-
>  meta-isar/conf/distro/debian-jessie.list           |   2 +
>  meta-isar/conf/distro/debian-stretch.conf          |  12 +-
>  meta-isar/conf/distro/debian-stretch.list          |   2 +
>  meta-isar/conf/distro/debian-wheezy.conf           |  14 ++-
>  meta-isar/conf/distro/debian-wheezy.list           |   2 +
>  meta-isar/conf/distro/raspbian-jessie.conf         |   9 +-
>  .../isar-bootstrap/files/isar-apt.conf             |   2 +
>  meta/recipes-core/isar-bootstrap/isar-bootstrap.bb | 128 +++++++++++++++++++++
>  9 files changed, 168 insertions(+), 17 deletions(-)
>  create mode 100644 meta-isar/conf/distro/debian-jessie.list
>  create mode 100644 meta-isar/conf/distro/debian-stretch.list
>  create mode 100644 meta-isar/conf/distro/debian-wheezy.list
>  create mode 100644 meta/recipes-core/isar-bootstrap/files/isar-apt.conf
>  create mode 100644 meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> 
> diff --git a/meta-isar/conf/distro/debian-jessie.conf b/meta-isar/conf/distro/debian-jessie.conf
> index 3e62b14..d7efa63 100644
> --- a/meta-isar/conf/distro/debian-jessie.conf
> +++ b/meta-isar/conf/distro/debian-jessie.conf
> @@ -1,16 +1,20 @@
> -# Debian jessie multistrap configuration
> +# Debian jessie distribution configuration
>  #
>  # This software is a part of ISAR.
>  # Copyright (C) 2015-2016 ilbers GmbH
>  
>  DISTRO_SUITE ?= "jessie"
> -DISTRO_COMPONENTS ?= "main contrib non-free"
> +# For bootstrap purposes still needed:
>  DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
> -DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_COMPONENTS ?= "main contrib non-free"
> +DISTRO_APT_SOURCES += "conf/distro/debian-jessie.list"
>  DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
> -DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> -DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
>  DISTRO_KERNELS ?= "486 4kc-malta 586 5kc-malta 686-pae amd64 arm64 armmp \
>      armmp-lpae ixp4xx kirkwood loongson-2e loongson-2f loongson-3 octeon \
>      orion5x powerpc powerpc-smp powerpc64 powerpc64le r4k-ip22 r5k-ip32 \
>      s390x sb1-bcm91250a versatile"
> +
> +# No longer needed:
> +DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> +DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
> diff --git a/meta-isar/conf/distro/debian-jessie.list b/meta-isar/conf/distro/debian-jessie.list
> new file mode 100644
> index 0000000..3b5cc59
> --- /dev/null
> +++ b/meta-isar/conf/distro/debian-jessie.list
> @@ -0,0 +1,2 @@
> +deb	http://ftp.de.debian.org/debian jessie-updates	main contrib non-free
> +deb	http://security.debian.org	jessie/updates	main contrib non-free
> diff --git a/meta-isar/conf/distro/debian-stretch.conf b/meta-isar/conf/distro/debian-stretch.conf
> index 5a85d52..26a08c5 100644
> --- a/meta-isar/conf/distro/debian-stretch.conf
> +++ b/meta-isar/conf/distro/debian-stretch.conf
> @@ -2,12 +2,16 @@
>  # Copyright (C) 2017 ilbers GmbH
>  
>  DISTRO_SUITE ?= "stretch"
> -DISTRO_COMPONENTS ?= "main contrib non-free"
> +# For bootstrap purposes still needed:
>  DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
> -DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_COMPONENTS ?= "main contrib non-free"
> +DISTRO_APT_SOURCES += "conf/distro/debian-stretch.list"
>  DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
> -DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> -DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
>  DISTRO_KERNELS ?= "4kc-malta 586 5kc-malta 686 686-pae amd64 arm64 armmp \
>      armmp-lpae kirkwood loongson-3 marvell octeon orion5x powerpc64le \
>      rt-686-pae rt-amd64 s390x"
> +
> +# No longer needed:
> +DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> +DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
> diff --git a/meta-isar/conf/distro/debian-stretch.list b/meta-isar/conf/distro/debian-stretch.list
> new file mode 100644
> index 0000000..842a859
> --- /dev/null
> +++ b/meta-isar/conf/distro/debian-stretch.list
> @@ -0,0 +1,2 @@
> +deb	http://ftp.de.debian.org/debian stretch-updates	main contrib non-free
> +deb	http://security.debian.org	stretch/updates	main contrib non-free
> diff --git a/meta-isar/conf/distro/debian-wheezy.conf b/meta-isar/conf/distro/debian-wheezy.conf
> index f5ab6eb..de7a10f 100644
> --- a/meta-isar/conf/distro/debian-wheezy.conf
> +++ b/meta-isar/conf/distro/debian-wheezy.conf
> @@ -1,14 +1,18 @@
> -# Debian wheezy multistrap configuration
> +# Debian wheezy distribution configuration
>  #
>  # This software is a part of ISAR.
>  # Copyright (C) 2015-2016 ilbers GmbH
>  
>  DISTRO_SUITE ?= "wheezy"
> -DISTRO_COMPONENTS ?= "main contrib non-free"
> +# For bootstrap purposes still needed:
>  DISTRO_APT_SOURCE ?= "http://ftp.debian.org/debian"
> -DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_COMPONENTS ?= "main contrib non-free"
> +DISTRO_APT_SOURCES += "conf/distro/debian-wheezy.list"
>  DISTRO_CONFIG_SCRIPT ?= "debian-configscript.sh"
> -DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> -DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
>  DISTRO_KERNELS ?= "486 686-pae amd64 rt-686-pae rt-amd64 versatile vexpress \
>      iop32x ixp4xx kirkwood mv78xx0 mx5 omap orion5x"
> +
> +# No longer needed:
> +DISTRO_APT_SOURCE_SEC ?= "http://security.debian.org/debian-security"
> +DISTRO_MULTICONF_BOOTSTRAP ?= "base updates security"
> +DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
> diff --git a/meta-isar/conf/distro/debian-wheezy.list b/meta-isar/conf/distro/debian-wheezy.list
> new file mode 100644
> index 0000000..2dbb112
> --- /dev/null
> +++ b/meta-isar/conf/distro/debian-wheezy.list
> @@ -0,0 +1,2 @@
> +deb	http://ftp.de.debian.org/debian wheezy-updates	main contrib non-free
> +deb	http://security.debian.org	wheezy/updates	main contrib non-free
> diff --git a/meta-isar/conf/distro/raspbian-jessie.conf b/meta-isar/conf/distro/raspbian-jessie.conf
> index c2a867d..0d66905 100644
> --- a/meta-isar/conf/distro/raspbian-jessie.conf
> +++ b/meta-isar/conf/distro/raspbian-jessie.conf
> @@ -1,13 +1,16 @@
> -# Raspbian stable multistrap configuration
> +# Raspbian stable distribution configuration
>  #
>  # This software is a part of ISAR.
>  # Copyright (C) 2015-2016 ilbers GmbH
>  
>  DISTRO_SUITE ?= "jessie"
> +# For bootstrap purposes still needed:
>  DISTRO_COMPONENTS ?= "main contrib non-free firmware"
>  DISTRO_APT_SOURCE ?= "http://archive.raspbian.org/raspbian"
> +DISTRO_CONFIG_SCRIPT?= "raspbian-configscript.sh"
> +DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv"
> +
> +# No longer needed:
>  DISTRO_APT_SOURCE_SEC ?= ""
> -DISTRO_CONFIG_SCRIPT ?= "raspbian-configscript.sh"
>  DISTRO_MULTICONF_BOOTSTRAP ?= "base"
>  DISTRO_MULTICONF_APTSOURCES ?= "${DISTRO_MULTICONF_BOOTSTRAP}"
> -DISTRO_KERNELS ?= "rpi rpi2 rpi-rpfv rpi2-rpfv"

DISTRO_APT_SOURCES missing, see also my patch in the other mail.

> diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt.conf b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf
> new file mode 100644
> index 0000000..c378775
> --- /dev/null
> +++ b/meta/recipes-core/isar-bootstrap/files/isar-apt.conf
> @@ -0,0 +1,2 @@
> +APT::Install-Recommends "0";
> +APT::Install-Suggests "0";
> \ No newline at end of file
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> new file mode 100644
> index 0000000..60da4b1
> --- /dev/null
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> @@ -0,0 +1,128 @@
> +# Minimal debian root file system
> +#
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2018
> +#
> +# SPDX-License-Identifier: MIT
> +
> +Description = "Minimal debian root file system"
> +
> +LICENSE = "gpl-2.0"
> +LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
> +FILESPATH_prepend := "${THISDIR}/files:"
> +SRC_URI = "file://isar-apt.conf"
> +PV = "1.0"
> +
> +WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
> +DEBOOTSTRAP ?= ""
> +ROOTFSDIR = "${WORKDIR}/rootfs"
> +APTPREFS = "${WORKDIR}/apt-preferences"
> +APTSRCS = "${WORKDIR}/apt-sources"
> +
> +python () {
> +    debootstrap = d.getVar("DEBOOTSTRAP", True)
> +    if not debootstrap:
> +        target = d.getVar("DISTRO_ARCH", True)
> +        machine = os.uname()[4]
> +        m = {
> +            "x86_64": ["i386", "amd64"],
> +            "x86": ["i386"],
> +            }
> +        if machine not in m or target not in m[machine]:
> +            debootstrap = "qemu-debootstrap"
> +        else:
> +            debootstrap = "debootstrap"
> +        d.setVar("DEBOOTSTRAP", debootstrap)
> +}
> +
> +def aggregate_files(d, file_list, file_out):
> +    import shutil
> +
> +    with open(file_out, "wb") as out_fd:
> +        for entry in file_list:
> +            entry_real = bb.parse.resolve_file(entry, d)
> +            with open(entry_real, "rb") as in_fd:
> +                 shutil.copyfileobj(in_fd, out_fd, 1024*1024*10)
> +            out_fd.write("\n".encode())
> +
> +def gen_components_argument(d):
> +    components = d.getVar("DISTRO_COMPONENTS", True)
> +    if components and components.strip():
> +        return "--components=%s" % ",".join(components.split())
> +    else:
> +        return ""
> +
> +do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> +do_bootstrap[dirs] = "${WORKDIR}"
> +do_bootstrap[vardeps] += "DISTRO_COMPONENTS"
> +do_bootstrap() {
> +    if [ -e "${ROOTFSDIR}" ]; then
> +       sudo umount -l "${ROOTFSDIR}/dev" || true
> +       sudo umount -l "${ROOTFSDIR}/proc" || true
> +       sudo rm -rf "${ROOTFSDIR}"
> +    fi
> +    E="${@bb.utils.export_proxies(d)}"
> +    sudo -E "${DEBOOTSTRAP}" --verbose \
> +                             --variant minbase \
> +                             --arch "${DISTRO_ARCH}" \
> +                             ${@ gen_components_argument(d)} \
> +                             "${DISTRO_SUITE}" \
> +                             "${ROOTFSDIR}" \
> +                             "${DISTRO_APT_SOURCE}"
> +}
> +addtask bootstrap before do_build after do_unpack
> +
> +# TODO: add stamp with checksum of preferences and source files 
> +# TODO: add mirror handling
> +do_apt_config_prepare[vardeps] += "\
> +                                   APTPREFS \
> +                                   DISTRO_APT_PREFERENCES \
> +                                   DEBDISTRONAME \
> +                                   APTSRCS \
> +                                   DISTRO_APT_SOURCES \
> +                                  "
> +python do_apt_config_prepare() {
> +    import textwrap
> +
> +    apt_preferences_out = d.getVar("APTPREFS", True)
> +    apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
> +                           ).split()
> +    aggregate_files(d, apt_preferences_list, apt_preferences_out)
> +
> +    apt_sources_out = d.getVar("APTSRCS", True)
> +    apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
> +
> +    aggregate_files(d, apt_sources_list, apt_sources_out)
> +}
> +addtask apt_config_prepare before do_build after do_bootstrap
> +
> +do_apt_config_install() {
> +    sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> +    sudo install -v -m644 "${APTPREFS}" \
> +                          "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> +    sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> +    sudo install -v -m644 "${APTSRCS}" \
> +                          "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> +    sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
> +    sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
> +                          "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> +}
> +addtask apt_config_install before do_build after do_apt_config_prepare
> +
> +do_apt_update() {
> +    sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
> +    sudo mount -t proc none ${ROOTFSDIR}/proc
> +
> +    E="${@bb.utils.export_proxies(d)}"
> +    export DEBIAN_FRONTEND=noninteractive
> +    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> +    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> +                                      -o Debug::pkgProblemResolver=yes
> +}
> +addtask apt_update before do_build after do_apt_config_install
> +
> +do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
> +do_deploy() {
> +    ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"

Don't we need to remove this on do_clean (CLEANFUNCS)?

> +}
> +addtask deploy before do_build after do_apt_update
> \ No newline at end of file
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux