From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6577660805416747008 X-Received: by 2002:adf:c58d:: with SMTP id m13-v6mr1547974wrg.9.1531752900171; Mon, 16 Jul 2018 07:55:00 -0700 (PDT) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a1c:e910:: with SMTP id q16-v6ls3297354wmc.2.gmail; Mon, 16 Jul 2018 07:54:59 -0700 (PDT) X-Google-Smtp-Source: AAOMgpcRWjF/cCccDJQ3umL9hatBIagtZ1LwmNqFSGjWfypSvILv+Z/ygN15PMpIphdsIoJNdus3 X-Received: by 2002:a1c:4b13:: with SMTP id y19-v6mr1520487wma.4.1531752899773; Mon, 16 Jul 2018 07:54:59 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1531752899; cv=none; d=google.com; s=arc-20160816; b=rZcHikqq8W4yqGvw2hq+51zAzw8RiX1iiXiBcgvhVMsK1aoEKptxYR4sBpKwriT9Jf /MZy0W7okmF7dyxqFFCx+auNOV1sR7I6jTLAVfG2q4CDpa7TL58C3CRJewZiqxuXqDbV KBqAL5GeFi053vIEUwjelBY8d3bg8iDzveUAzI+12CNWQepMk7I09gKowymVxfgV9L8Z 5mHq/rkI+dkdUBjmojPKFkmsddo1MuflNVT4ljVB2OUny/G6aXKWQqrGrX1n9/Nag2mf KtmfZsvyybxWcKoRCOtwN3MgfU/x4UgcZ38xIPk6HmnLbPlawtBIuXPkXlOxErN40G+z guSw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=message-id:date:subject:to:from:arc-authentication-results; bh=HRM7wi44QgPDQK5rR6LuYnGflpRWPL2sKHBtgs4AgDM=; b=iAI9mlHo9ngYfHYQjArmC4NqEUg2axP379racA9Sgq858qnQbxAXIgKMliBoFZdEcB JF0WG8odAAPl77VRZaAA+02n2NOM2pMGpZTn60r4xAL5wy4HT5BFaXe+LTlqpnLzLfVy fgneECAUBoraV/YGW3weh+ODpDBUxs5TNip+vGNsO588Bcse7n8jKOLsV1U5FGAugFFj bn9vMT81rz7zxhnv52cv49iFbsx/clvJn64eZ2pyvOUVvxlQ2GqKYe8CG/tCXU43IIei 9ozORhFG2GNvxvrZxvByYEgS+AgIAcGsuI3ygqCgecnaLhASMoQDm1IkkV+r8B9p+5AH QdGg== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: best guess record for domain of mosipov@ilbers.de designates 85.214.62.211 as permitted sender) smtp.mailfrom=mosipov@ilbers.de Return-Path: Received: from aqmola.ilbers.de (aqmola.ilbers.de. [85.214.62.211]) by gmr-mx.google.com with ESMTPS id j6-v6si62137wmh.2.2018.07.16.07.54.59 for (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 16 Jul 2018 07:54:59 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of mosipov@ilbers.de designates 85.214.62.211 as permitted sender) client-ip=85.214.62.211; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: best guess record for domain of mosipov@ilbers.de designates 85.214.62.211 as permitted sender) smtp.mailfrom=mosipov@ilbers.de Received: from azat.i.ilbers.de (host-80-81-17-52.static.customer.m-online.net [80.81.17.52]) (authenticated bits=0) by aqmola.ilbers.de (8.14.4/8.14.4/Debian-4+deb7u1) with ESMTP id w6GEsnxP021498 (version=TLSv1/SSLv3 cipher=AES128-SHA256 bits=128 verify=NOT) for ; Mon, 16 Jul 2018 16:54:58 +0200 From: "Maxim Yu. Osipov" To: isar-users@googlegroups.com Subject: [PATCH v2] isar-bootstrap: Eliminate multiple debootstraps for the same distro/host Date: Mon, 16 Jul 2018 16:54:49 +0200 Message-Id: <20180716145449.5772-1-mosipov@ilbers.de> X-Mailer: git-send-email 2.11.0 X-TUID: Sa3jSYGC/YRh After applying this patch only single debootstrap for particular platform is created and pointed by corresponding link under DEPLOY_DIR_IMAGE. For proper locking of parallel builds shell tasks apt_config_install, set_locale, setup_chroot, apt_update and deploy were collapsed to common debootstrap helper. Due to problems in bitbake with shell functions expansion under quotes these functions were substituted by their bodies. Signed-off-by: Maxim Yu. Osipov --- meta/classes/image.bbclass | 2 +- .../isar-bootstrap/isar-bootstrap-host.bb | 57 +++------ .../isar-bootstrap/isar-bootstrap-target.bb | 52 +++------ .../recipes-core/isar-bootstrap/isar-bootstrap.inc | 127 ++++++++++++++------- meta/recipes-devtools/buildchroot/buildchroot.bb | 2 +- meta/recipes-devtools/sdkchroot/sdkchroot.bb | 2 +- 6 files changed, 119 insertions(+), 123 deletions(-) diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index 7935b69..20eb3fd 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass @@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}" inherit ${IMAGE_TYPE} do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}" -do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy" +do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap" do_rootfs() { die "No root filesystem function defined, please implement in your recipe" diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb index cca0984..55696ea 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb @@ -8,13 +8,15 @@ Description = "Minimal host Debian root file system" WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}" +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}" +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock" include isar-bootstrap.inc inherit isar-bootstrap-helper -do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" +do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" -do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" +do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" do_apt_config_prepare[dirs] = "${WORKDIR}" do_apt_config_prepare[vardeps] += "\ APTPREFS \ @@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\ DEBDISTRONAME \ APTSRCS \ HOST_DISTRO_APT_SOURCES \ + DEPLOY_ISAR_BOOTSTRAP \ " python do_apt_config_prepare() { - apt_preferences_out = d.getVar("APTPREFS", True) - apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or "" - ).split() - aggregate_files(d, apt_preferences_list, apt_preferences_out) + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)): + apt_preferences_out = d.getVar("APTPREFS", True) + apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or "" + ).split() + aggregate_files(d, apt_preferences_list, apt_preferences_out) - apt_sources_out = d.getVar("APTSRCS", True) - apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split() + apt_sources_out = d.getVar("APTSRCS", True) + apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split() - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) } -addtask apt_config_prepare before do_build after do_unpack - -do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" - +addtask apt_config_prepare before do_bootstrap after do_unpack do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES" -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS" do_bootstrap() { - if [ -e "${ROOTFSDIR}" ]; then - sudo umount -l "${ROOTFSDIR}/dev" || true - sudo umount -l "${ROOTFSDIR}/proc" || true - sudo rm -rf "${ROOTFSDIR}" - fi - E="${@bb.utils.export_proxies(d)}" - sudo -E "${DEBOOTSTRAP}" --verbose \ - --variant=minbase \ - --include=locales \ - ${@get_distro_components_argument(d, True)} \ - ${DEBOOTSTRAP_KEYRING} \ - "${@get_distro_suite(d, True)}" \ - "${ROOTFSDIR}" \ - "${@get_distro_source(d, True)}" + isar_bootstrap --host } addtask bootstrap before do_build after do_generate_keyring - -do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}" -do_deploy() { - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}" -} -addtask deploy before do_build after do_apt_update - -do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" - -CLEANFUNCS = "clean_deploy" -clean_deploy() { - rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}" -} diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb index 77b5d8d..5752b14 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb @@ -8,6 +8,8 @@ Description = "Minimal target Debian root file system" WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}" +DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}" +ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock" include isar-bootstrap.inc @@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\ DEBDISTRONAME \ APTSRCS \ DISTRO_APT_SOURCES \ + DEPLOY_ISAR_BOOTSTRAP \ " python do_apt_config_prepare() { - apt_preferences_out = d.getVar("APTPREFS", True) - apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or "" - ).split() - aggregate_files(d, apt_preferences_list, apt_preferences_out) + if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)): + apt_preferences_out = d.getVar("APTPREFS", True) + apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or "" + ).split() + aggregate_files(d, apt_preferences_list, apt_preferences_out) - apt_sources_out = d.getVar("APTSRCS", True) - apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split() + apt_sources_out = d.getVar("APTSRCS", True) + apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split() - aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) + aggregate_aptsources_list(d, apt_sources_list, apt_sources_out) } -addtask apt_config_prepare before do_build after do_unpack - -do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" +addtask apt_config_prepare before do_bootstrap after do_unpack do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" do_bootstrap[vardeps] += "DISTRO_APT_SOURCES" -do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS" do_bootstrap() { - if [ -e "${ROOTFSDIR}" ]; then - sudo umount -l "${ROOTFSDIR}/dev" || true - sudo umount -l "${ROOTFSDIR}/proc" || true - sudo rm -rf "${ROOTFSDIR}" - fi - E="${@bb.utils.export_proxies(d)}" - sudo -E "${DEBOOTSTRAP}" --verbose \ - --variant=minbase \ - --arch="${DISTRO_ARCH}" \ - --include=locales \ - ${@get_distro_components_argument(d, False)} \ - ${DEBOOTSTRAP_KEYRING} \ - "${@get_distro_suite(d, False)}" \ - "${ROOTFSDIR}" \ - "${@get_distro_source(d, False)}" + isar_bootstrap } addtask bootstrap before do_build after do_generate_keyring -do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" -do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}" -do_deploy() { - ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}" -} -addtask deploy before do_build after do_apt_update - -do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}" - -CLEANFUNCS = "clean_deploy" -clean_deploy() { - rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}" -} diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index 18ec72f..8afd470 100644 --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources" APTKEYFILES = "" APTKEYRING = "${WORKDIR}/apt-keyring.gpg" DEBOOTSTRAP_KEYRING = "" +DEPLOY_ISAR_BOOTSTRAP ?= "" python () { from urllib.parse import urlparse @@ -149,52 +150,100 @@ do_generate_keyring() { } addtask generate_keyring before do_build after do_unpack -do_set_locale() { - sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale" - sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen" - sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen -} -addtask set_locale after do_bootstrap - -do_setup_chroot() { - sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh" - sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" -} -addtask setup_chroot before do_build after do_bootstrap def get_host_release(): import platform rel = platform.release() return rel -do_apt_config_install() { - sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d" - sudo install -v -m644 "${APTPREFS}" \ - "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap" - sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d" - sudo install -v -m644 "${APTSRCS}" \ - "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list" - sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list" - sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d" - sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \ - "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf" - - if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then - sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \ - "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf" - fi -} -addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare -do_apt_update() { - sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev - sudo mount -t proc none ${ROOTFSDIR}/proc - - E="${@bb.utils.export_proxies(d)}" - export DEBIAN_FRONTEND=noninteractive - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y - sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \ +do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS" +do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}" + +isar_bootstrap() { + IS_HOST="" + while true; do + case "$1" in + --host) IS_HOST=1 ;; + -*) bbfatal "$0: invalid option specified: $1" ;; + *) break ;; + esac + shift + done + sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\ + if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then + if [ -e "${ROOTFSDIR}" ]; then + sudo umount -l "${ROOTFSDIR}/dev" || true + sudo umount -l "${ROOTFSDIR}/proc" || true + sudo rm -rf "${ROOTFSDIR}" + fi + E="${@bb.utils.export_proxies(d)}" + if [ ${IS_HOST} ]; then + sudo -E "${DEBOOTSTRAP}" --verbose \ + --variant=minbase \ + --include=locales \ + ${@get_distro_components_argument(d, True)} \ + ${DEBOOTSTRAP_KEYRING} \ + "${@get_distro_suite(d, True)}" \ + "${ROOTFSDIR}" \ + "${@get_distro_source(d, True)}" + + else + sudo -E "${DEBOOTSTRAP}" --verbose \ + --variant=minbase \ + --arch="${DISTRO_ARCH}" \ + --include=locales \ + ${@get_distro_components_argument(d, False)} \ + ${DEBOOTSTRAP_KEYRING} \ + "${@get_distro_suite(d, False)}" \ + "${ROOTFSDIR}" \ + "${@get_distro_source(d, False)}" + fi + + # Install apt config + sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d" + sudo install -v -m644 "${APTPREFS}" \ + "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap" + sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d" + sudo install -v -m644 "${APTSRCS}" \ + "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list" + sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list" + sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d" + sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \ + "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf" + + if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then + sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \ + "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf" + fi + + # Set locale + sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale" + + sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen" + sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen + + # setup chroot + sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh" + sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}" + + # update APT + sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev + sudo mount -t proc none ${ROOTFSDIR}/proc + + E="${@bb.utils.export_proxies(d)}" + export DEBIAN_FRONTEND=noninteractive + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y + sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \ -o Debug::pkgProblemResolver=yes + + # Finalize debootstrap by setting the link in deploy + ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}" + fi" +} + +CLEANFUNCS = "clean_deploy" +clean_deploy() { + rm -f "${DEPLOY_ISAR_BOOTSTRAP}" } -addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb index e0d2302..7ad24f1 100644 --- a/meta/recipes-devtools/buildchroot/buildchroot.bb +++ b/meta/recipes-devtools/buildchroot/buildchroot.bb @@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \ ${BUILDCHROOT_DIR}/isar-apt \ ${BUILDCHROOT_DIR}/downloads \ ${BUILDCHROOT_DIR}/home/builder" -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy" +do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap" do_build() { setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL} diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb index 70aa4ce..cfec95c 100644 --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb @@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}" do_build[root_cleandirs] = "${S} \ ${S}/isar-apt" -do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy" +do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap" do_build() { -- 2.11.0