* [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host
@ 2018-07-16 14:54 Maxim Yu. Osipov
2018-07-17 15:24 ` Maxim Yu. Osipov
0 siblings, 1 reply; 2+ messages in thread
From: Maxim Yu. Osipov @ 2018-07-16 14:54 UTC (permalink / raw)
To: isar-users
After applying this patch only single debootstrap for
particular platform is created and pointed by corresponding
link under DEPLOY_DIR_IMAGE.
For proper locking of parallel builds shell tasks
apt_config_install, set_locale, setup_chroot, apt_update
and deploy were collapsed to common debootstrap helper.
Due to problems in bitbake with shell functions expansion
under quotes these functions were substituted by their
bodies.
Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>
---
meta/classes/image.bbclass | 2 +-
.../isar-bootstrap/isar-bootstrap-host.bb | 57 +++------
.../isar-bootstrap/isar-bootstrap-target.bb | 52 +++------
.../recipes-core/isar-bootstrap/isar-bootstrap.inc | 127 ++++++++++++++-------
meta/recipes-devtools/buildchroot/buildchroot.bb | 2 +-
meta/recipes-devtools/sdkchroot/sdkchroot.bb | 2 +-
6 files changed, 119 insertions(+), 123 deletions(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 7935b69..20eb3fd 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}"
inherit ${IMAGE_TYPE}
do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
-do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
do_rootfs() {
die "No root filesystem function defined, please implement in your recipe"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
index cca0984..55696ea 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
@@ -8,13 +8,15 @@
Description = "Minimal host Debian root file system"
WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock"
include isar-bootstrap.inc
inherit isar-bootstrap-helper
-do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
do_apt_config_prepare[dirs] = "${WORKDIR}"
do_apt_config_prepare[vardeps] += "\
APTPREFS \
@@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\
DEBDISTRONAME \
APTSRCS \
HOST_DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
"
python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS", True)
- apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
+ if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+ apt_preferences_out = d.getVar("APTPREFS", True)
+ apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
- apt_sources_out = d.getVar("APTSRCS", True)
- apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
+ apt_sources_out = d.getVar("APTSRCS", True)
+ apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
}
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
+addtask apt_config_prepare before do_bootstrap after do_unpack
do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap() {
- if [ -e "${ROOTFSDIR}" ]; then
- sudo umount -l "${ROOTFSDIR}/dev" || true
- sudo umount -l "${ROOTFSDIR}/proc" || true
- sudo rm -rf "${ROOTFSDIR}"
- fi
- E="${@bb.utils.export_proxies(d)}"
- sudo -E "${DEBOOTSTRAP}" --verbose \
- --variant=minbase \
- --include=locales \
- ${@get_distro_components_argument(d, True)} \
- ${DEBOOTSTRAP_KEYRING} \
- "${@get_distro_suite(d, True)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d, True)}"
+ isar_bootstrap --host
}
addtask bootstrap before do_build after do_generate_keyring
-
-do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}"
-}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
index 77b5d8d..5752b14 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
@@ -8,6 +8,8 @@
Description = "Minimal target Debian root file system"
WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock"
include isar-bootstrap.inc
@@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\
DEBDISTRONAME \
APTSRCS \
DISTRO_APT_SOURCES \
+ DEPLOY_ISAR_BOOTSTRAP \
"
python do_apt_config_prepare() {
- apt_preferences_out = d.getVar("APTPREFS", True)
- apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
- ).split()
- aggregate_files(d, apt_preferences_list, apt_preferences_out)
+ if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+ apt_preferences_out = d.getVar("APTPREFS", True)
+ apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
+ ).split()
+ aggregate_files(d, apt_preferences_list, apt_preferences_out)
- apt_sources_out = d.getVar("APTSRCS", True)
- apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+ apt_sources_out = d.getVar("APTSRCS", True)
+ apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
- aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+ aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
}
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+addtask apt_config_prepare before do_bootstrap after do_unpack
do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap() {
- if [ -e "${ROOTFSDIR}" ]; then
- sudo umount -l "${ROOTFSDIR}/dev" || true
- sudo umount -l "${ROOTFSDIR}/proc" || true
- sudo rm -rf "${ROOTFSDIR}"
- fi
- E="${@bb.utils.export_proxies(d)}"
- sudo -E "${DEBOOTSTRAP}" --verbose \
- --variant=minbase \
- --arch="${DISTRO_ARCH}" \
- --include=locales \
- ${@get_distro_components_argument(d, False)} \
- ${DEBOOTSTRAP_KEYRING} \
- "${@get_distro_suite(d, False)}" \
- "${ROOTFSDIR}" \
- "${@get_distro_source(d, False)}"
+ isar_bootstrap
}
addtask bootstrap before do_build after do_generate_keyring
-do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
- ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
- rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
-}
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 18ec72f..8afd470 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources"
APTKEYFILES = ""
APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
DEBOOTSTRAP_KEYRING = ""
+DEPLOY_ISAR_BOOTSTRAP ?= ""
python () {
from urllib.parse import urlparse
@@ -149,52 +150,100 @@ do_generate_keyring() {
}
addtask generate_keyring before do_build after do_unpack
-do_set_locale() {
- sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
- sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
- sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-}
-addtask set_locale after do_bootstrap
-
-do_setup_chroot() {
- sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
- sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
-}
-addtask setup_chroot before do_build after do_bootstrap
def get_host_release():
import platform
rel = platform.release()
return rel
-do_apt_config_install() {
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
- sudo install -v -m644 "${APTPREFS}" \
- "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
- sudo install -v -m644 "${APTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
- sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
- sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
- sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
- "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
-
- if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
- sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
- "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
- fi
-}
-addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
-do_apt_update() {
- sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
- sudo mount -t proc none ${ROOTFSDIR}/proc
-
- E="${@bb.utils.export_proxies(d)}"
- export DEBIAN_FRONTEND=noninteractive
- sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
- sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
+do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}"
+
+isar_bootstrap() {
+ IS_HOST=""
+ while true; do
+ case "$1" in
+ --host) IS_HOST=1 ;;
+ -*) bbfatal "$0: invalid option specified: $1" ;;
+ *) break ;;
+ esac
+ shift
+ done
+ sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
+ if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
+ if [ -e "${ROOTFSDIR}" ]; then
+ sudo umount -l "${ROOTFSDIR}/dev" || true
+ sudo umount -l "${ROOTFSDIR}/proc" || true
+ sudo rm -rf "${ROOTFSDIR}"
+ fi
+ E="${@bb.utils.export_proxies(d)}"
+ if [ ${IS_HOST} ]; then
+ sudo -E "${DEBOOTSTRAP}" --verbose \
+ --variant=minbase \
+ --include=locales \
+ ${@get_distro_components_argument(d, True)} \
+ ${DEBOOTSTRAP_KEYRING} \
+ "${@get_distro_suite(d, True)}" \
+ "${ROOTFSDIR}" \
+ "${@get_distro_source(d, True)}"
+
+ else
+ sudo -E "${DEBOOTSTRAP}" --verbose \
+ --variant=minbase \
+ --arch="${DISTRO_ARCH}" \
+ --include=locales \
+ ${@get_distro_components_argument(d, False)} \
+ ${DEBOOTSTRAP_KEYRING} \
+ "${@get_distro_suite(d, False)}" \
+ "${ROOTFSDIR}" \
+ "${@get_distro_source(d, False)}"
+ fi
+
+ # Install apt config
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+ sudo install -v -m644 "${APTPREFS}" \
+ "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+ sudo install -v -m644 "${APTSRCS}" \
+ "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+ sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+ sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
+ sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
+ "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
+
+ if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
+ sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
+ "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
+ fi
+
+ # Set locale
+ sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
+
+ sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
+ sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
+
+ # setup chroot
+ sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
+ sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+
+ # update APT
+ sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
+ sudo mount -t proc none ${ROOTFSDIR}/proc
+
+ E="${@bb.utils.export_proxies(d)}"
+ export DEBIAN_FRONTEND=noninteractive
+ sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
+ sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
-o Debug::pkgProblemResolver=yes
+
+ # Finalize debootstrap by setting the link in deploy
+ ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+ fi"
+}
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+ rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
}
-addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index e0d2302..7ad24f1 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
${BUILDCHROOT_DIR}/isar-apt \
${BUILDCHROOT_DIR}/downloads \
${BUILDCHROOT_DIR}/home/builder"
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
do_build() {
setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL}
diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
index 70aa4ce..cfec95c 100644
--- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
+++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
@@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
do_build[root_cleandirs] = "${S} \
${S}/isar-apt"
-do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy"
+do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap"
do_build() {
--
2.11.0
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host
2018-07-16 14:54 [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host Maxim Yu. Osipov
@ 2018-07-17 15:24 ` Maxim Yu. Osipov
0 siblings, 0 replies; 2+ messages in thread
From: Maxim Yu. Osipov @ 2018-07-17 15:24 UTC (permalink / raw)
To: isar-users
On 07/16/2018 05:54 PM, Maxim Yu. Osipov wrote:
> After applying this patch only single debootstrap for
> particular platform is created and pointed by corresponding
> link under DEPLOY_DIR_IMAGE.
>
> For proper locking of parallel builds shell tasks
> apt_config_install, set_locale, setup_chroot, apt_update
> and deploy were collapsed to common debootstrap helper.
> Due to problems in bitbake with shell functions expansion
> under quotes these functions were substituted by their
> bodies.
Applied to the 'next'.
> Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>
> ---
> meta/classes/image.bbclass | 2 +-
> .../isar-bootstrap/isar-bootstrap-host.bb | 57 +++------
> .../isar-bootstrap/isar-bootstrap-target.bb | 52 +++------
> .../recipes-core/isar-bootstrap/isar-bootstrap.inc | 127 ++++++++++++++-------
> meta/recipes-devtools/buildchroot/buildchroot.bb | 2 +-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 2 +-
> 6 files changed, 119 insertions(+), 123 deletions(-)
>
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index 7935b69..20eb3fd 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}"
> inherit ${IMAGE_TYPE}
>
> do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
> -do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
> +do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
>
> do_rootfs() {
> die "No root filesystem function defined, please implement in your recipe"
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> index cca0984..55696ea 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb
> @@ -8,13 +8,15 @@
> Description = "Minimal host Debian root file system"
>
> WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}"
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
> +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock"
>
> include isar-bootstrap.inc
> inherit isar-bootstrap-helper
>
> -do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> +do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>
> -do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> +do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> do_apt_config_prepare[dirs] = "${WORKDIR}"
> do_apt_config_prepare[vardeps] += "\
> APTPREFS \
> @@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\
> DEBDISTRONAME \
> APTSRCS \
> HOST_DISTRO_APT_SOURCES \
> + DEPLOY_ISAR_BOOTSTRAP \
> "
> python do_apt_config_prepare() {
> - apt_preferences_out = d.getVar("APTPREFS", True)
> - apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
> - ).split()
> - aggregate_files(d, apt_preferences_list, apt_preferences_out)
> + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
> + apt_preferences_out = d.getVar("APTPREFS", True)
> + apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
> + ).split()
> + aggregate_files(d, apt_preferences_list, apt_preferences_out)
>
> - apt_sources_out = d.getVar("APTSRCS", True)
> - apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
> + apt_sources_out = d.getVar("APTSRCS", True)
> + apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
>
> - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> }
> -addtask apt_config_prepare before do_build after do_unpack
> -
> -do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -
> +addtask apt_config_prepare before do_bootstrap after do_unpack
>
> do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
> -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> do_bootstrap() {
> - if [ -e "${ROOTFSDIR}" ]; then
> - sudo umount -l "${ROOTFSDIR}/dev" || true
> - sudo umount -l "${ROOTFSDIR}/proc" || true
> - sudo rm -rf "${ROOTFSDIR}"
> - fi
> - E="${@bb.utils.export_proxies(d)}"
> - sudo -E "${DEBOOTSTRAP}" --verbose \
> - --variant=minbase \
> - --include=locales \
> - ${@get_distro_components_argument(d, True)} \
> - ${DEBOOTSTRAP_KEYRING} \
> - "${@get_distro_suite(d, True)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d, True)}"
> + isar_bootstrap --host
> }
> addtask bootstrap before do_build after do_generate_keyring
> -
> -do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
> -do_deploy() {
> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
> -}
> -addtask deploy before do_build after do_apt_update
> -
> -do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> -
> -CLEANFUNCS = "clean_deploy"
> -clean_deploy() {
> - rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}"
> -}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> index 77b5d8d..5752b14 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb
> @@ -8,6 +8,8 @@
> Description = "Minimal target Debian root file system"
>
> WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
> +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
> +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock"
>
> include isar-bootstrap.inc
>
> @@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\
> DEBDISTRONAME \
> APTSRCS \
> DISTRO_APT_SOURCES \
> + DEPLOY_ISAR_BOOTSTRAP \
> "
> python do_apt_config_prepare() {
> - apt_preferences_out = d.getVar("APTPREFS", True)
> - apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
> - ).split()
> - aggregate_files(d, apt_preferences_list, apt_preferences_out)
> + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
> + apt_preferences_out = d.getVar("APTPREFS", True)
> + apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
> + ).split()
> + aggregate_files(d, apt_preferences_list, apt_preferences_out)
>
> - apt_sources_out = d.getVar("APTSRCS", True)
> - apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
> + apt_sources_out = d.getVar("APTSRCS", True)
> + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
>
> - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
> }
> -addtask apt_config_prepare before do_build after do_unpack
> -
> -do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> +addtask apt_config_prepare before do_bootstrap after do_unpack
>
> do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
> -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> do_bootstrap() {
> - if [ -e "${ROOTFSDIR}" ]; then
> - sudo umount -l "${ROOTFSDIR}/dev" || true
> - sudo umount -l "${ROOTFSDIR}/proc" || true
> - sudo rm -rf "${ROOTFSDIR}"
> - fi
> - E="${@bb.utils.export_proxies(d)}"
> - sudo -E "${DEBOOTSTRAP}" --verbose \
> - --variant=minbase \
> - --arch="${DISTRO_ARCH}" \
> - --include=locales \
> - ${@get_distro_components_argument(d, False)} \
> - ${DEBOOTSTRAP_KEYRING} \
> - "${@get_distro_suite(d, False)}" \
> - "${ROOTFSDIR}" \
> - "${@get_distro_source(d, False)}"
> + isar_bootstrap
> }
> addtask bootstrap before do_build after do_generate_keyring
>
> -do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
> -do_deploy() {
> - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
> -}
> -addtask deploy before do_build after do_apt_update
> -
> -do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -
> -CLEANFUNCS = "clean_deploy"
> -clean_deploy() {
> - rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
> -}
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 18ec72f..8afd470 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources"
> APTKEYFILES = ""
> APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
> DEBOOTSTRAP_KEYRING = ""
> +DEPLOY_ISAR_BOOTSTRAP ?= ""
>
> python () {
> from urllib.parse import urlparse
> @@ -149,52 +150,100 @@ do_generate_keyring() {
> }
> addtask generate_keyring before do_build after do_unpack
>
> -do_set_locale() {
> - sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
>
> - sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> - sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> -}
> -addtask set_locale after do_bootstrap
> -
> -do_setup_chroot() {
> - sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> - sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> -}
> -addtask setup_chroot before do_build after do_bootstrap
>
> def get_host_release():
> import platform
> rel = platform.release()
> return rel
>
> -do_apt_config_install() {
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> - sudo install -v -m644 "${APTPREFS}" \
> - "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> - sudo install -v -m644 "${APTSRCS}" \
> - "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> - sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> - sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
> - sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
> - "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> -
> - if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
> - sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
> - "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
> - fi
> -}
> -addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
>
> -do_apt_update() {
> - sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
> - sudo mount -t proc none ${ROOTFSDIR}/proc
> -
> - E="${@bb.utils.export_proxies(d)}"
> - export DEBIAN_FRONTEND=noninteractive
> - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> +do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> +do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}"
> +
> +isar_bootstrap() {
> + IS_HOST=""
> + while true; do
> + case "$1" in
> + --host) IS_HOST=1 ;;
> + -*) bbfatal "$0: invalid option specified: $1" ;;
> + *) break ;;
> + esac
> + shift
> + done
> + sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
> + if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
> + if [ -e "${ROOTFSDIR}" ]; then
> + sudo umount -l "${ROOTFSDIR}/dev" || true
> + sudo umount -l "${ROOTFSDIR}/proc" || true
> + sudo rm -rf "${ROOTFSDIR}"
> + fi
> + E="${@bb.utils.export_proxies(d)}"
> + if [ ${IS_HOST} ]; then
> + sudo -E "${DEBOOTSTRAP}" --verbose \
> + --variant=minbase \
> + --include=locales \
> + ${@get_distro_components_argument(d, True)} \
> + ${DEBOOTSTRAP_KEYRING} \
> + "${@get_distro_suite(d, True)}" \
> + "${ROOTFSDIR}" \
> + "${@get_distro_source(d, True)}"
> +
> + else
> + sudo -E "${DEBOOTSTRAP}" --verbose \
> + --variant=minbase \
> + --arch="${DISTRO_ARCH}" \
> + --include=locales \
> + ${@get_distro_components_argument(d, False)} \
> + ${DEBOOTSTRAP_KEYRING} \
> + "${@get_distro_suite(d, False)}" \
> + "${ROOTFSDIR}" \
> + "${@get_distro_source(d, False)}"
> + fi
> +
> + # Install apt config
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> + sudo install -v -m644 "${APTPREFS}" \
> + "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
> + sudo install -v -m644 "${APTSRCS}" \
> + "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
> + sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
> + sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
> + sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
> + "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> +
> + if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
> + sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
> + "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
> + fi
> +
> + # Set locale
> + sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
> +
> + sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
> + sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
> +
> + # setup chroot
> + sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
> + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
> +
> + # update APT
> + sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
> + sudo mount -t proc none ${ROOTFSDIR}/proc
> +
> + E="${@bb.utils.export_proxies(d)}"
> + export DEBIAN_FRONTEND=noninteractive
> + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> -o Debug::pkgProblemResolver=yes
> +
> + # Finalize debootstrap by setting the link in deploy
> + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
> + fi"
> +}
> +
> +CLEANFUNCS = "clean_deploy"
> +clean_deploy() {
> + rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
> }
> -addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
> index e0d2302..7ad24f1 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
> @@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
> ${BUILDCHROOT_DIR}/isar-apt \
> ${BUILDCHROOT_DIR}/downloads \
> ${BUILDCHROOT_DIR}/home/builder"
> -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
> +do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
>
> do_build() {
> setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL}
> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> index 70aa4ce..cfec95c 100644
> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> @@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
> do_build[root_cleandirs] = "${S} \
> ${S}/isar-apt"
>
> -do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy"
> +do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap"
>
> do_build() {
>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2018-07-17 15:24 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-07-16 14:54 [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host Maxim Yu. Osipov
2018-07-17 15:24 ` Maxim Yu. Osipov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox