From: "Maxim Yu. Osipov" <mosipov@ilbers.de>
To: isar-users@googlegroups.com
Subject: Re: [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host
Date: Tue, 17 Jul 2018 18:24:06 +0300 [thread overview]
Message-ID: <30887f64-d0d8-ee2f-cddc-27d4fff98f99@ilbers.de> (raw)
In-Reply-To: <20180716145449.5772-1-mosipov@ilbers.de>
On 07/16/2018 05:54 PM, Maxim Yu. Osipov wrote:
> After applying this patch only single debootstrap for
> particular platform is created and pointed by corresponding
> link under DEPLOY_DIR_IMAGE.
>
> For proper locking of parallel builds shell tasks
> apt_config_install, set_locale, setup_chroot, apt_update
> and deploy were collapsed to common debootstrap helper.
> Due to problems in bitbake with shell functions expansion
> under quotes these functions were substituted by their
> bodies.
Applied to the 'next'.
> Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>
> ---
> meta/classes/image.bbclass | 2 +-
> .../isar-bootstrap/isar-bootstrap-host.bb | 57 +++------
> .../isar-bootstrap/isar-bootstrap-target.bb | 52 +++------
> .../recipes-core/isar-bootstrap/isar-bootstrap.inc | 127 ++++++++++++++-------
> meta/recipes-devtools/buildchroot/buildchroot.bb | 2 +-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 2 +-
> 6 files changed, 119 insertions(+), 123 deletions(-)
>
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index 7935b69..20eb3fd 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}"
> inherit ${IMAGE_TYPE}
>
> do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
> -do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
> +do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
>
> do_rootfs() {
> die "No root filesystem function defined, please implement in your recipe"
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> index cca0984..55696ea 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> @@ -8,13 +8,15 @@
> Description = "Minimal host Debian root file system"
>
> WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}"
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
> +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock"
>
> include isar-bootstrap.inc
> inherit isar-bootstrap-helper
>
> -do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> +do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>
> -do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> +do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> do_apt_config_prepare[dirs] = "${WORKDIR}"
> do_apt_config_prepare[vardeps] += "\
> APTPREFS \
> @@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\
> DEBDISTRONAME \
> APTSRCS \
> HOST_DISTRO_APT_SOURCES \
> + DEPLOY_ISAR_BOOTSTRAP \
> "
> python do_apt_config_prepare() {
> - apt_preferences_out = d.getVar("APTPREFS", True)
> - apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
> - ).split()
> - aggregate_files(d, apt_preferences_list, apt_preferences_out)
> + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
> + apt_preferences_out = d.getVar("APTPREFS", True)
> + apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
> + ).split()
> + aggregate_files(d, apt_preferences_list, apt_preferences_out)
>
> - apt_sources_out = d.getVar("APTSRCS", True)
> - apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
> + apt_sources_out = d.getVar("APTSRCS", True)
> + apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
>
> - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> }
> -addtask apt_config_prepare before do_build after do_unpack
> -
> -do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -
> +addtask apt_config_prepare before do_bootstrap after do_unpack
>
> do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
> -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> do_bootstrap() {
> - if [ -e "${ROOTFSDIR}" ]; then
> - sudo umount -l "${ROOTFSDIR}/dev" || true
> - sudo umount -l "${ROOTFSDIR}/proc" || true
> - sudo rm -rf "${ROOTFSDIR}"
> - fi
> - E="${@bb.utils.export_proxies(d)}"
> - sudo -E "${DEBOOTSTRAP}" --verbose \
> - --variant=minbase \
> - --include=locales \
> - ${@get_distro_components_argument(d, True)} \
> - ${DEBOOTSTRAP_KEYRING} \
> - "${@get_distro_suite(d, True)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d, True)}"
> + isar_bootstrap --host
> }
> addtask bootstrap before do_build after do_generate_keyring
> -
> -do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
> -do_deploy() {
> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
> -}
> -addtask deploy before do_build after do_apt_update
> -
> -do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -
> -CLEANFUNCS = "clean_deploy"
> -clean_deploy() {
> - rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}"
> -}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> index 77b5d8d..5752b14 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> @@ -8,6 +8,8 @@
> Description = "Minimal target Debian root file system"
>
> WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
> +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock"
>
> include isar-bootstrap.inc
>
> @@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\
> DEBDISTRONAME \
> APTSRCS \
> DISTRO_APT_SOURCES \
> + DEPLOY_ISAR_BOOTSTRAP \
> "
> python do_apt_config_prepare() {
> - apt_preferences_out = d.getVar("APTPREFS", True)
> - apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
> - ).split()
> - aggregate_files(d, apt_preferences_list, apt_preferences_out)
> + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
> + apt_preferences_out = d.getVar("APTPREFS", True)
> + apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
> + ).split()
> + aggregate_files(d, apt_preferences_list, apt_preferences_out)
>
> - apt_sources_out = d.getVar("APTSRCS", True)
> - apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
> + apt_sources_out = d.getVar("APTSRCS", True)
> + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
>
> - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> }
> -addtask apt_config_prepare before do_build after do_unpack
> -
> -do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> +addtask apt_config_prepare before do_bootstrap after do_unpack
>
> do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
> -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> do_bootstrap() {
> - if [ -e "${ROOTFSDIR}" ]; then
> - sudo umount -l "${ROOTFSDIR}/dev" || true
> - sudo umount -l "${ROOTFSDIR}/proc" || true
> - sudo rm -rf "${ROOTFSDIR}"
> - fi
> - E="${@bb.utils.export_proxies(d)}"
> - sudo -E "${DEBOOTSTRAP}" --verbose \
> - --variant=minbase \
> - --arch="${DISTRO_ARCH}" \
> - --include=locales \
> - ${@get_distro_components_argument(d, False)} \
> - ${DEBOOTSTRAP_KEYRING} \
> - "${@get_distro_suite(d, False)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d, False)}"
> + isar_bootstrap
> }
> addtask bootstrap before do_build after do_generate_keyring
>
> -do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
> -do_deploy() {
> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
> -}
> -addtask deploy before do_build after do_apt_update
> -
> -do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -
> -CLEANFUNCS = "clean_deploy"
> -clean_deploy() {
> - rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
> -}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 18ec72f..8afd470 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources"
> APTKEYFILES = ""
> APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
> DEBOOTSTRAP_KEYRING = ""
> +DEPLOY_ISAR_BOOTSTRAP ?= ""
>
> python () {
> from urllib.parse import urlparse
> @@ -149,52 +150,100 @@ do_generate_keyring() {
> }
> addtask generate_keyring before do_build after do_unpack
>
> -do_set_locale() {
> - sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
>
> - sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> - sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> -}
> -addtask set_locale after do_bootstrap
> -
> -do_setup_chroot() {
> - sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> - sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> -}
> -addtask setup_chroot before do_build after do_bootstrap
>
> def get_host_release():
> import platform
> rel = platform.release()
> return rel
>
> -do_apt_config_install() {
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> - sudo install -v -m644 "${APTPREFS}" \
> - "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> - sudo install -v -m644 "${APTSRCS}" \
> - "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> - sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
> - sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
> - "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> -
> - if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
> - sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
> - "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
> - fi
> -}
> -addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
>
> -do_apt_update() {
> - sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
> - sudo mount -t proc none ${ROOTFSDIR}/proc
> -
> - E="${@bb.utils.export_proxies(d)}"
> - export DEBIAN_FRONTEND=noninteractive
> - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> +do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}"
> +
> +isar_bootstrap() {
> + IS_HOST=""
> + while true; do
> + case "$1" in
> + --host) IS_HOST=1 ;;
> + -*) bbfatal "$0: invalid option specified: $1" ;;
> + *) break ;;
> + esac
> + shift
> + done
> + sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
> + if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
> + if [ -e "${ROOTFSDIR}" ]; then
> + sudo umount -l "${ROOTFSDIR}/dev" || true
> + sudo umount -l "${ROOTFSDIR}/proc" || true
> + sudo rm -rf "${ROOTFSDIR}"
> + fi
> + E="${@bb.utils.export_proxies(d)}"
> + if [ ${IS_HOST} ]; then
> + sudo -E "${DEBOOTSTRAP}" --verbose \
> + --variant=minbase \
> + --include=locales \
> + ${@get_distro_components_argument(d, True)} \
> + ${DEBOOTSTRAP_KEYRING} \
> + "${@get_distro_suite(d, True)}" \
> + "${ROOTFSDIR}" \
> + "${@get_distro_source(d, True)}"
> +
> + else
> + sudo -E "${DEBOOTSTRAP}" --verbose \
> + --variant=minbase \
> + --arch="${DISTRO_ARCH}" \
> + --include=locales \
> + ${@get_distro_components_argument(d, False)} \
> + ${DEBOOTSTRAP_KEYRING} \
> + "${@get_distro_suite(d, False)}" \
> + "${ROOTFSDIR}" \
> + "${@get_distro_source(d, False)}"
> + fi
> +
> + # Install apt config
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> + sudo install -v -m644 "${APTPREFS}" \
> + "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> + sudo install -v -m644 "${APTSRCS}" \
> + "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> + sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
> + sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
> + "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> +
> + if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
> + sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
> + "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
> + fi
> +
> + # Set locale
> + sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
> +
> + sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> + sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> +
> + # setup chroot
> + sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> +
> + # update APT
> + sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
> + sudo mount -t proc none ${ROOTFSDIR}/proc
> +
> + E="${@bb.utils.export_proxies(d)}"
> + export DEBIAN_FRONTEND=noninteractive
> + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> -o Debug::pkgProblemResolver=yes
> +
> + # Finalize debootstrap by setting the link in deploy
> + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> + fi"
> +}
> +
> +CLEANFUNCS = "clean_deploy"
> +clean_deploy() {
> + rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
> }
> -addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
> index e0d2302..7ad24f1 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
> @@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
> ${BUILDCHROOT_DIR}/isar-apt \
> ${BUILDCHROOT_DIR}/downloads \
> ${BUILDCHROOT_DIR}/home/builder"
> -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
> +do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
>
> do_build() {
> setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL}
> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> index 70aa4ce..cfec95c 100644
> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> @@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> do_build[root_cleandirs] = "${S} \
> ${S}/isar-apt"
>
> -do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy"
> +do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap"
>
> do_build() {
>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
prev parent reply other threads:[~2018-07-17 15:24 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-07-16 14:54 Maxim Yu. Osipov
2018-07-17 15:24 ` Maxim Yu. Osipov [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=30887f64-d0d8-ee2f-cddc-27d4fff98f99@ilbers.de \
--to=mosipov@ilbers.de \
--cc=isar-users@googlegroups.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox