From: "Maxim Yu. Osipov" <mosipov@ilbers.de>
To: isar-users@googlegroups.com
Subject: [PATCH] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host
Date: Fri, 13 Jul 2018 13:22:59 +0200 [thread overview]
Message-ID: <20180713112259.19647-1-mosipov@ilbers.de> (raw)
After applying this patch only single debootstrap for
particular platform is created and pointed by corresponding
link uder DEPLOY_DIR_IMAGE.
For proper locking of parallel builds shell tasks
apt_config_install, set_locale, setup_chroot, apt_update
and deploy were collapsed to common debotstrap helper.
Due to problems in bitbake with shell functions expansion
under quotes these functions were substituted by their
bodies.
Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>
---
meta/classes/image.bbclass | 2 +-
.../isar-bootstrap/isar-bootstrap-host.bb | 57 +++------
.../isar-bootstrap/isar-bootstrap-target.bb | 52 +++------
.../recipes-core/isar-bootstrap/isar-bootstrap.inc | 127 ++++++++++++++-------
meta/recipes-devtools/buildchroot/buildchroot.bb | 2 +-
meta/recipes-devtools/sdkchroot/sdkchroot.bb | 2 +-
6 files changed, 119 insertions(+), 123 deletions(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 7935b69..20eb3fd 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}"
inherit ${IMAGE_TYPE}
do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
-do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
do_rootfs() {
die "No root filesystem function defined, please implement in your recipe"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
index cca0984..3996d8d 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
@@ -8,13 +8,15 @@
Description = "Minimal host Debian root file system"
WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock"
include isar-bootstrap.inc
inherit isar-bootstrap-helper
-do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
do_apt_config_prepare[dirs] = "${WORKDIR}"
do_apt_config_prepare[vardeps] += "\
APTPREFS \
@@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\
DEBDISTRONAME \
APTSRCS \
HOST_DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
"
python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS", True)
- apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
+ if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+ apt_preferences_out = d.getVar("APTPREFS", True)
+ apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
- apt_sources_out = d.getVar("APTSRCS", True)
- apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
+ apt_sources_out = d.getVar("APTSRCS", True)
+ apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
}
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
+addtask apt_config_prepare before do_bootstrap after do_unpack
do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap() {
- if [ -e "${ROOTFSDIR}" ]; then
- sudo umount -l "${ROOTFSDIR}/dev" || true
- sudo umount -l "${ROOTFSDIR}/proc" || true
- sudo rm -rf "${ROOTFSDIR}"
- fi
- E="${@bb.utils.export_proxies(d)}"
- sudo -E "${DEBOOTSTRAP}" --verbose \
- --variant=minbase \
- --include=locales \
- ${@get_distro_components_argument(d, True)} \
- ${DEBOOTSTRAP_KEYRING} \
- "${@get_distro_suite(d, True)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d, True)}"
+ isar_bootstrap --host
}
addtask bootstrap before do_build after do_generate_keyring
-
-do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}"
-}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
index 77b5d8d..afa856d 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
@@ -8,6 +8,8 @@
Description = "Minimal target Debian root file system"
WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock"
include isar-bootstrap.inc
@@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\
DEBDISTRONAME \
APTSRCS \
DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
"
python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS", True)
- apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
+ if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+ apt_preferences_out = d.getVar("APTPREFS", True)
+ apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
- apt_sources_out = d.getVar("APTSRCS", True)
- apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+ apt_sources_out = d.getVar("APTSRCS", True)
+ apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
}
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+addtask apt_config_prepare before do_bootstrap after do_unpack
do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap() {
- if [ -e "${ROOTFSDIR}" ]; then
- sudo umount -l "${ROOTFSDIR}/dev" || true
- sudo umount -l "${ROOTFSDIR}/proc" || true
- sudo rm -rf "${ROOTFSDIR}"
- fi
- E="${@bb.utils.export_proxies(d)}"
- sudo -E "${DEBOOTSTRAP}" --verbose \
- --variant=minbase \
- --arch="${DISTRO_ARCH}" \
- --include=locales \
- ${@get_distro_components_argument(d, False)} \
- ${DEBOOTSTRAP_KEYRING} \
- "${@get_distro_suite(d, False)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d, False)}"
+ isar_bootstrap
}
addtask bootstrap before do_build after do_generate_keyring
-do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
-}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 18ec72f..8afd470 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources"
APTKEYFILES = ""
APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
DEBOOTSTRAP_KEYRING = ""
+DEPLOY_ISAR_BOOTSTRAP ?= ""
python () {
from urllib.parse import urlparse
@@ -149,52 +150,100 @@ do_generate_keyring() {
}
addtask generate_keyring before do_build after do_unpack
-do_set_locale() {
- sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
- sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-}
-addtask set_locale after do_bootstrap
-
-do_setup_chroot() {
- sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
-}
-addtask setup_chroot before do_build after do_bootstrap
def get_host_release():
import platform
rel = platform.release()
return rel
-do_apt_config_install() {
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- sudo install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- sudo install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
- sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
- "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
-
- if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
- sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
- "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
- fi
-}
-addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
-do_apt_update() {
- sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
- sudo mount -t proc none ${ROOTFSDIR}/proc
-
- E="${@bb.utils.export_proxies(d)}"
- export DEBIAN_FRONTEND=noninteractive
- sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
- sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
+do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}"
+
+isar_bootstrap() {
+ IS_HOST=""
+ while true; do
+ case "$1" in
+ --host) IS_HOST=1 ;;
+ -*) bbfatal "$0: invalid option specified: $1" ;;
+ *) break ;;
+ esac
+ shift
+ done
+ sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
+ if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
+ if [ -e "${ROOTFSDIR}" ]; then
+ sudo umount -l "${ROOTFSDIR}/dev" || true
+ sudo umount -l "${ROOTFSDIR}/proc" || true
+ sudo rm -rf "${ROOTFSDIR}"
+ fi
+ E="${@bb.utils.export_proxies(d)}"
+ if [ ${IS_HOST} ]; then
+ sudo -E "${DEBOOTSTRAP}" --verbose \
+ --variant=minbase \
+ --include=locales \
+ ${@get_distro_components_argument(d, True)} \
+ ${DEBOOTSTRAP_KEYRING} \
+ "${@get_distro_suite(d, True)}" \
+ "${ROOTFSDIR}" \
+ "${@get_distro_source(d, True)}"
+
+ else
+ sudo -E "${DEBOOTSTRAP}" --verbose \
+ --variant=minbase \
+ --arch="${DISTRO_ARCH}" \
+ --include=locales \
+ ${@get_distro_components_argument(d, False)} \
+ ${DEBOOTSTRAP_KEYRING} \
+ "${@get_distro_suite(d, False)}" \
+ "${ROOTFSDIR}" \
+ "${@get_distro_source(d, False)}"
+ fi
+
+ # Install apt config
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+ sudo install -v -m644 "${APTPREFS}" \
+ "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+ sudo install -v -m644 "${APTSRCS}" \
+ "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+ sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
+ sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
+ "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
+
+ if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
+ sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
+ "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
+ fi
+
+ # Set locale
+ sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
+
+ sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
+ sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
+
+ # setup chroot
+ sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+
+ # update APT
+ sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
+ sudo mount -t proc none ${ROOTFSDIR}/proc
+
+ E="${@bb.utils.export_proxies(d)}"
+ export DEBIAN_FRONTEND=noninteractive
+ sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
+ sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
-o Debug::pkgProblemResolver=yes
+
+ # Finalize debootstrap by setting the link in deploy
+ ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+ fi"
+}
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+ rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
}
-addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index e0d2302..7ad24f1 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
${BUILDCHROOT_DIR}/isar-apt \
${BUILDCHROOT_DIR}/downloads \
${BUILDCHROOT_DIR}/home/builder"
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
do_build() {
setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL}
diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
index 70aa4ce..cfec95c 100644
--- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
+++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
@@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
do_build[root_cleandirs] = "${S} \
${S}/isar-apt"
-do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy"
+do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap"
do_build() {
--
2.11.0
next reply other threads:[~2018-07-13 11:23 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-07-13 11:22 Maxim Yu. Osipov [this message]
2018-07-13 14:33 ` Jan Kiszka
2018-07-16 7:18 ` Maxim Yu. Osipov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180713112259.19647-1-mosipov@ilbers.de \
--to=mosipov@ilbers.de \
--cc=isar-users@googlegroups.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox