* [PATCH v4 1/8] meta: remove transient package support
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline claudius.heine.ext
` (8 subsequent siblings)
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Transient packages where meant for configuring images via packages,
which proved to not be fit for purpose.
Packages are meant to be shared across images, configuring it this way
would mean that every possible configuration permutation would need to
result in a separate package just so that no image accidentally installs
a configuration package meant for a different build/image.
Instead images need to be configured while its specific root file system
is created using the new root file system creation pipeline.
Signed-off-by: Claudius Heine <ch@denx.de>
---
meta/classes/image.bbclass | 4 +-
meta/classes/isar-bootstrap-helper.bbclass | 13 +---
.../isar-cfg-localepurge/files/postinst | 15 ----
.../isar-cfg-localepurge.bb | 70 -------------------
.../isar-cfg-rootpw/files/postinst.tmpl | 19 -----
.../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 -----
6 files changed, 2 insertions(+), 138 deletions(-)
delete mode 100644 meta/recipes-support/isar-cfg-localepurge/files/postinst
delete mode 100644 meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
delete mode 100644 meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
delete mode 100644 meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 3eb261f..d77626c 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -36,9 +36,7 @@ def cfg_script(d):
FILESPATH =. "${LAYERDIR_core}/conf/distro:"
SRC_URI += "${@ cfg_script(d) }"
-DEPENDS += "${IMAGE_INSTALL} ${IMAGE_TRANSIENT_PACKAGES}"
-
-IMAGE_TRANSIENT_PACKAGES += "isar-cfg-localepurge isar-cfg-rootpw"
+DEPENDS += "${IMAGE_INSTALL}"
ISAR_RELEASE_CMD_DEFAULT = "git -C ${LAYERDIR_core} describe --tags --dirty --match 'v[0-9].[0-9]*'"
ISAR_RELEASE_CMD ?= "${ISAR_RELEASE_CMD_DEFAULT}"
diff --git a/meta/classes/isar-bootstrap-helper.bbclass b/meta/classes/isar-bootstrap-helper.bbclass
index 8612be2..bbc6322 100644
--- a/meta/classes/isar-bootstrap-helper.bbclass
+++ b/meta/classes/isar-bootstrap-helper.bbclass
@@ -5,8 +5,6 @@
#
# SPDX-License-Identifier: MIT
-IMAGE_TRANSIENT_PACKAGES ??= ""
-
def reverse_bb_array(d, varname):
array = d.getVar(varname, True)
if array is None:
@@ -113,19 +111,10 @@ setup_root_file_system() {
sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
fi
sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES \
- ${IMAGE_TRANSIENT_PACKAGES}
+ /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
[ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
sudo -E chroot "$ROOTFSDIR" \
/usr/bin/apt-get ${APT_ARGS} $PACKAGES
- for pkg in ${IMAGE_TRANSIENT_PACKAGES}; do
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get ${APT_ARGS} $pkg
- done
- for pkg in ${@reverse_bb_array(d, "IMAGE_TRANSIENT_PACKAGES") or ""}; do
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get purge --yes $pkg
- done
if [ ${CLEAN} ]; then
if [ ${KEEP_APT_CACHE} -eq 1 ]; then
mkdir -p ${WORKDIR}/apt_cache
diff --git a/meta/recipes-support/isar-cfg-localepurge/files/postinst b/meta/recipes-support/isar-cfg-localepurge/files/postinst
deleted file mode 100644
index 9dfd6f9..0000000
--- a/meta/recipes-support/isar-cfg-localepurge/files/postinst
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-set -e
-
-cat /usr/lib/isar-cfg-localepurge/locale.gen \
- >> /etc/locale.gen
-cat /usr/lib/isar-cfg-localepurge/locale.default \
- > /etc/default/locale
-cat /usr/lib/isar-cfg-localepurge/locale.nopurge \
- > /etc/locale.nopurge
-
-debconf-set-selections /usr/lib/isar-cfg-localepurge/locale.debconf
-
-dpkg-reconfigure -f noninteractive locales
-
-localepurge
diff --git a/meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb b/meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
deleted file mode 100644
index 62b4b2d..0000000
--- a/meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
+++ /dev/null
@@ -1,70 +0,0 @@
-# This software is a part of ISAR.
-
-DESCRIPTION = "Isar configuration package for locale and localepurge"
-MAINTAINER = "isar-users <isar-users@googlegroups.com>"
-DEBIAN_DEPENDS = "localepurge"
-
-SRC_URI = "file://postinst"
-
-inherit dpkg-raw
-
-LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
- en_US ISO-8859-1\n"
-LOCALE_DEFAULT ?= "en_US.UTF-8"
-
-def get_locale_gen(d):
- locale_gen = d.getVar("LOCALE_GEN", True) or ""
- return '\n'.join(sorted(set(i.strip()
- for i in locale_gen.split('\\n')
- if i.strip())))
-
-def get_dc_locale_gen(d):
- locale_gen = d.getVar("LOCALE_GEN", True) or ""
- return ', '.join(sorted(set(i.strip()
- for i in locale_gen.split('\\n')
- if i.strip())))
-
-def get_nopurge(d):
- locale_gen = d.getVar("LOCALE_GEN", True) or ""
- return '\n'.join(sorted(set(i.strip()
- for j in locale_gen.split('\\n')
- if j.strip()
- for i in (j.split()[0].split("_")[0],
- j.split()[0].split(".")[0],
- j.split()[0]))))
-
-do_gen_config[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_gen_config() {
- cat<<-__EOF__ > ${WORKDIR}/locale.gen
- ${@get_locale_gen(d)}
- __EOF__
- cat<<-__EOF__ > ${WORKDIR}/locale.debconf
- locales locales/locales_to_be_generated multiselect ${@get_dc_locale_gen(d)}
- locales locales/default_environment_locale select ${LOCALE_DEFAULT}
- __EOF__
- cat<<-__EOF__ > ${WORKDIR}/locale.default
- LANG=${LOCALE_DEFAULT}
- __EOF__
- cat<<-__EOF__ > ${WORKDIR}/locale.nopurge
- #USE_DPKG
- MANDELETE
- DONTBOTHERNEWLOCALE
- #SHOWFREEDSPACE
- #QUICKNDIRTYCALC
- #VERBOSE
- ${@get_nopurge(d)}
- __EOF__
-}
-addtask gen_config after do_unpack before do_install
-
-do_install() {
- install -v -d ${D}/usr/lib/${PN}
- install -v -m 644 ${WORKDIR}/locale.debconf \
- ${D}/usr/lib/${PN}/locale.debconf
- install -v -m 644 ${WORKDIR}/locale.gen \
- ${D}/usr/lib/${PN}/locale.gen
- install -v -m 644 ${WORKDIR}/locale.default \
- ${D}/usr/lib/${PN}/locale.default
- install -v -m 644 ${WORKDIR}/locale.nopurge \
- ${D}/usr/lib/${PN}/locale.nopurge
-}
diff --git a/meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl b/meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
deleted file mode 100644
index ca08a41..0000000
--- a/meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-set -e
-
-if ! grep -q 'root:\*:' /etc/shadow; then
- echo "ERROR:isar-cfg-rootpw: root password was set by a different package" >&2
- exit -1
-fi
-
-if [ -n '${CFG_ROOT_PW}' ]; then
- echo 'root:${CFG_ROOT_PW}' | chpasswd -e
-else
- passwd -d root
-fi
-
-if [ '${CFG_ROOT_LOCKED}' = "1" ]; then
- # Lock the account after setting the password, since unlocking it at some
- # point later would set it to the back to the previous one.
- passwd -l root
-fi
diff --git a/meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb b/meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
deleted file mode 100644
index adee3b5..0000000
--- a/meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
+++ /dev/null
@@ -1,19 +0,0 @@
-# This software is a part of ISAR.
-
-DESCRIPTION = "Isar configuration package for root password"
-MAINTAINER = "isar-users <isar-users@googlegroups.com>"
-DEBIAN_DEPENDS = "passwd"
-
-SRC_URI = "file://postinst.tmpl"
-
-TEMPLATE_FILES = "postinst.tmpl"
-TEMPLATE_VARS = "CFG_ROOT_PW CFG_ROOT_LOCKED"
-
-CFG_ROOT_PW ??= ""
-CFG_ROOT_LOCKED ??= "0"
-
-inherit dpkg-raw
-
-do_install() {
- echo "intentionally left blank"
-}
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 1/8] meta: remove transient package support claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-24 12:49 ` Maxim Yu. Osipov
2019-08-14 16:00 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline Jan Kiszka
2019-05-23 14:55 ` [PATCH v4 3/8] meta/classes: add image-locales-extension class claudius.heine.ext
` (7 subsequent siblings)
9 siblings, 2 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
This removes the isar-bootstrap-helper.bbclass and integrates its
functionality into the rootfs.bbclass.
It introduces the rootfs_install task that runs through multiple
functions from command lists in order to allow easy integration of
custom functions.
The introduced functions are, in general order of execution:
rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
rootfs_configure_*: basic configuration of the root file system,
before any custom packages where installed. (`/dev`, `/proc`, etc.
are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
rootfs_do_mount: mount standard directories and repositories into the
rootfs
do_rootfs_install_*: downloading and installing packages. Functions
are added to `ROOTFS_INSTALL_COMMAND`.
Signed-off-by: Claudius Heine <ch@denx.de>
---
meta/classes/image.bbclass | 24 +--
meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
.../isar-bootstrap/isar-bootstrap.inc | 5 +
.../buildchroot/buildchroot-host.bb | 16 +-
.../buildchroot/buildchroot-target.bb | 2 -
.../buildchroot/buildchroot.inc | 26 ++-
meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
8 files changed, 228 insertions(+), 197 deletions(-)
delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index d77626c..99eea92 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -51,8 +51,10 @@ image_do_mounts() {
buildchroot_do_mounts
}
-inherit isar-bootstrap-helper
-ROOTFS_FEATURES += "finalize-rootfs"
+ROOTFSDIR = "${IMAGE_ROOTFS}"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
+
inherit rootfs
inherit image-sdk-extension
inherit image-cache-extension
@@ -119,8 +121,10 @@ python set_image_size () {
d.setVarFlag('ROOTFS_SIZE', 'export', '1')
}
-do_image_gen_fstab() {
- cat > ${WORKDIR}/fstab << EOF
+ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
+image_configure_fstab[weight] = "2"
+image_configure_fstab() {
+ sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
# Begin /etc/fstab
/dev/root / auto defaults 0 0
proc /proc proc nosuid,noexec,nodev 0 0
@@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs mode=0755,nosuid 0 0
# End /etc/fstab
EOF
}
-addtask image_gen_fstab before do_rootfs_install
-
-do_rootfs_install[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
-do_rootfs_install[deptask] = "do_deploy_deb"
-do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
- ${IMAGE_ROOTFS}/isar-apt"
-do_rootfs_install() {
- setup_root_file_system --clean --keep-apt-cache \
- --fstab "${WORKDIR}/fstab" \
- "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
-}
-addtask rootfs_install before do_build after do_unpack
do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
do_copy_boot_files() {
diff --git a/meta/classes/isar-bootstrap-helper.bbclass b/meta/classes/isar-bootstrap-helper.bbclass
deleted file mode 100644
index bbc6322..0000000
--- a/meta/classes/isar-bootstrap-helper.bbclass
+++ /dev/null
@@ -1,132 +0,0 @@
-# Helper functions for using isar-bootstrap
-#
-# This software is a part of ISAR.
-# Copyright (c) Siemens AG, 2018
-#
-# SPDX-License-Identifier: MIT
-
-def reverse_bb_array(d, varname):
- array = d.getVar(varname, True)
- if array is None:
- return None
- array = reversed(array.split())
- return " ".join(i for i in array)
-
-
-update_etc_os_release() {
- OS_RELEASE_BUILD_ID=""
- OS_RELEASE_VARIANT=""
- while true; do
- case "$1" in
- --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
- --variant) OS_RELEASE_VARIANT=$2; shift ;;
- -*) bbfatal "$0: invalid option specified: $1" ;;
- *) break ;;
- esac
- shift
- done
- ROOTFSDIR="$1"
-
- if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
- sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
- echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
- sudo tee -a ${ROOTFSDIR}/etc/os-release
- fi
- if [ -n "${OS_RELEASE_VARIANT}" ]; then
- sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
- echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
- sudo tee -a ${ROOTFSDIR}/etc/os-release
- fi
-}
-
-setup_root_file_system() {
- CLEAN=""
- COPYISARAPT=""
- FSTAB=""
- ROOTFS_ARCH="${DISTRO_ARCH}"
- ROOTFS_DISTRO="${DISTRO}"
- while true; do
- case "$1" in
- --clean) CLEAN=1 ;;
- --copyisarapt) COPYISARAPT=1 ;;
- --fstab) FSTAB=$2; shift ;;
- --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
- --host-distro) ROOTFS_DISTRO="$2"; shift ;;
- --keep-apt-cache) KEEP_APT_CACHE=1 ;;
- -*) bbfatal "$0: invalid option specified: $1" ;;
- *) break ;;
- esac
- shift
- done
- ROOTFSDIR="$1"
- shift
- PACKAGES="$@"
- APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
- CLEAN_FILES="${ROOTFSDIR}/etc/hostname ${ROOTFSDIR}/etc/resolv.conf"
-
- sudo cp -Trpfx \
- "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
- "$ROOTFSDIR"
- [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee "$ROOTFSDIR/etc/fstab"
-
- echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main" | \
- sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list" >/dev/null
-
- echo "Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
- sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
-
- if [ ${COPYISARAPT} ]; then
- sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
- else
- sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
- fi
-
- if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- fi
-
- sudo mount --rbind /dev ${ROOTFSDIR}/dev
- sudo mount --make-rslave ${ROOTFSDIR}/dev
- sudo mount -t proc none $ROOTFSDIR/proc
- sudo mount --rbind /sys ${ROOTFSDIR}/sys
- sudo mount --make-rslave ${ROOTFSDIR}/sys
-
- # Refresh /etc/resolv.conf
- sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
-
- # Install packages:
- E="${@ bb.utils.export_proxies(d)}"
- export DEBIAN_FRONTEND=noninteractive
- # To avoid Perl locale warnings:
- export LANG=C
- export LANGUAGE=C
- export LC_ALL=C
- sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
- -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
- -o Dir::Etc::sourceparts="-" \
- -o APT::Get::List-Cleanup="0"
- # Add multiarch for cross-target
- if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
- sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
- sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
- fi
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
- [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get ${APT_ARGS} $PACKAGES
- if [ ${CLEAN} ]; then
- if [ ${KEEP_APT_CACHE} -eq 1 ]; then
- mkdir -p ${WORKDIR}/apt_cache
- sudo find ${ROOTFSDIR}/var/cache/apt/archives \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t ${WORKDIR}/apt_cache '{}' '+'
- sudo chown -R $(whoami) ${WORKDIR}/apt_cache
- fi
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get autoremove --purge --yes
- sudo -E chroot "$ROOTFSDIR" \
- /usr/bin/apt-get clean
- sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
- sudo rm -rf "$ROOTFSDIR/chroot-setup.sh" "$ROOTFSDIR/var/lib/apt/lists/"*
- fi
-}
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 45b0350..0fc8ba8 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,14 +1,178 @@
# This software is a part of ISAR.
# Copyright (c) Siemens AG, 2019
+ROOTFS_ARCH ?= "${DISTRO_ARCH}"
+ROOTFS_DISTRO ?= "${DISTRO}"
+ROOTFS_PACKAGES ?= ""
+
# Features of the rootfs creation:
# available features are:
+# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
+# 'clean-package-cache' - delete package cache from rootfs
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
ROOTFS_FEATURES ?= ""
+ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
+
+ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
+
+# Useful environment variables:
+export E = "${@ bb.utils.export_proxies(d)}"
+export DEBIAN_FRONTEND = "noninteractive"
+# To avoid Perl locale warnings:
+export LANG = "C"
+export LANGUAGE = "C"
+export LC_ALL = "C"
+
+rootfs_do_mounts[weight] = "3"
+rootfs_do_mounts() {
+ sudo -s <<'EOSUDO'
+ mountpoint -q '${ROOTFSDIR}/dev' || \
+ mount --rbind /dev '${ROOTFSDIR}/dev'
+ mount --make-rslave '${ROOTFSDIR}/dev'
+ mountpoint -q '${ROOTFSDIR}/proc' || \
+ mount -t proc none '${ROOTFSDIR}/proc'
+ mountpoint -q '${ROOTFSDIR}/sys' || \
+ mount --rbind /sys '${ROOTFSDIR}/sys'
+ mount --make-rslave '${ROOTFSDIR}/sys'
+
+ # Mount isar-apt if the directory does not exist or if it is empty
+ # This prevents overwriting something that was copied there
+ if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
+ [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth 1 | wc -l)" = "0" ]
+ then
+ mkdir -p '${ROOTFSDIR}/isar-apt'
+ mountpoint -q '${ROOTFSDIR}/isar-apt' || \
+ mount --bind '${REPO_ISAR_DIR}/${DISTRO}' '${ROOTFSDIR}/isar-apt'
+ fi
+
+ # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
+ if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}" = 'True' ]
+ then
+ mkdir -p '${ROOTFSDIR}/base-apt'
+ mountpoint -q '${ROOTFSDIR}/base-apt' || \
+ mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
+ fi
+
+EOSUDO
+}
+
+rootfs_do_qemu() {
+ if [ '${@repr(d.getVar('ROOTFS_ARCH') == d.getVar('HOST_ARCH'))}' = 'False' ]
+ then
+ test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
+ sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static' '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
+ fi
+}
+
+BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
+BOOTSTRAP_SRC_${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
+
+rootfs_prepare[weight] = "25"
+rootfs_prepare(){
+ sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
+}
+
+ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
+rootfs_configure_isar_apt[weight] = "2"
+rootfs_configure_isar_apt() {
+ sudo -s <<'EOSUDO'
+
+ mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
+ echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
+ '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
+
+ mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
+ cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
+Package: *
+Pin: release n=${DEBDISTRONAME}
+Pin-Priority: 1000
+EOF
+EOSUDO
+}
+
+ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
+rootfs_install_pkgs_update[weight] = "5"
+rootfs_install_pkgs_update() {
+ sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
+ -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::sourceparts="-" \
+ -o APT::Get::List-Cleanup="0"
+}
+ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
+rootfs_install_resolvconf[weight] = "1"
+rootfs_install_resolvconf() {
+ sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
+}
+
+ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
+rootfs_install_pkgs_download[weight] = "600"
+rootfs_install_pkgs_download() {
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
+}
+
+ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
+rootfs_install_clean_files[weight] = "2"
+rootfs_install_clean_files() {
+ sudo -E chroot '${ROOTFSDIR}' \
+ /bin/rm -f ${ROOTFS_CLEAN_FILES}
+}
+
+ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
+rootfs_install_pkgs_install[weight] = "8000"
+rootfs_install_pkgs_install() {
+ sudo -E chroot "${ROOTFSDIR}" \
+ /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
+}
+
+do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
+do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[deptask] = "do_deploy_deb"
+python do_rootfs_install() {
+ configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
+ install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or "").split()
+
+ # Mount after configure commands, so that they have time to copy
+ # 'isar-apt' (sdkchroot):
+ cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
+
+ stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
+ for i in cmds]
+
+ progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights)
+
+ for cmd in cmds:
+ progress_reporter.next_stage()
+ bb.build.exec_func(cmd, d)
+ progress_reporter.finish()
+}
+addtask rootfs_install before do_rootfs_postprocess after do_unpack
+
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
+rootfs_postprocess_copy_package_cache() {
+ mkdir -p '${WORKDIR}/apt_cache'
+ sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
+ -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
+ me="$(id -u):$(id -g)"
+ sudo chown -R "$me" '${WORKDIR}/apt_cache'
+}
+
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
+rootfs_postprocess_clean_package_cache() {
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/bin/apt-get clean
+ sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
+}
+
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs', 'rootfs_postprocess_finalize', '', d)}"
rootfs_postprocess_finalize() {
sudo -s <<'EOSUDO'
+ test -e "${ROOTFSDIR}/chroot-setup.sh" && \
+ "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
+ rm -f "${ROOTFSDIR}/chroot-setup.sh"
+
test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
find "${ROOTFSDIR}/usr/bin" \
-maxdepth 1 -name 'qemu-*-static' -type f -delete
@@ -42,6 +206,12 @@ EOSUDO
do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
python do_rootfs_postprocess() {
+ # Take care that its correctly mounted:
+ bb.build.exec_func('rootfs_do_mounts', d)
+ # Take care that qemu-*-static is available, since it could have been
+ # removed on a previous execution of this task:
+ bb.build.exec_func('rootfs_do_qemu', d)
+
cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
if cmds is None or not cmds.strip():
return
@@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
for cmd in cmds:
bb.build.exec_func(cmd, d)
}
-addtask rootfs_postprocess before do_rootfs after do_rootfs_install
+addtask rootfs_postprocess before do_rootfs
python do_rootfs() {
"""Virtual task"""
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index b740c57..f8741c2 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -327,6 +327,11 @@ isar_bootstrap() {
mount --make-rslave ${ROOTFSDIR}/sys
export DEBIAN_FRONTEND=noninteractive
+
+ if [ ${IS_HOST} ]; then
+ chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
+ fi
+
chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
-o Debug::pkgProblemResolver=yes
diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
index 73cd548..121992c 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
@@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
require buildchroot.inc
+ROOTFS_ARCH = "${HOST_ARCH}"
+ROOTFS_DISTRO = "${HOST_DISTRO}"
BUILDCHROOT_PREINSTALL ?= "make \
debhelper \
@@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
automake \
devscripts \
equivs \
- libc6:${DISTRO_ARCH}"
-
-# According to the wiki page:
-# https://wiki.debian.org/CrossToolchains
-BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
- crossbuild-essential-armhf"
-BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
- crossbuild-essential-arm64"
-
-
-PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-host:do_bootstrap"
+ libc6:${DISTRO_ARCH} \
+ crossbuild-essential-${DISTRO_ARCH}"
diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
index 20f4d23..5a01258 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
@@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
devscripts \
equivs \
adduser"
-
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
index ca9ce51..40fc8b0 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.inc
+++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
@@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
file://deps.sh"
PV = "1.0"
-inherit isar-bootstrap-helper
+inherit rootfs
BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
+ROOTFSDIR = "${BUILDCHROOT_DIR}"
+ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
+ROOTFS_CLEAN_FILES = ""
+
+rootfs_do_mounts_append() {
+ sudo -s <<'EOSUDO'
+ mkdir -p '${BUILDCHROOT_DIR}/downloads'
+ mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
+ mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
+EOSUDO
+}
-do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
- ${BUILDCHROOT_DIR}/isar-apt \
- ${BUILDCHROOT_DIR}/downloads \
- ${BUILDCHROOT_DIR}/home/builder"
-
-do_build() {
- setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR} ${BUILDCHROOT_PREINSTALL}
-
+ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
+buildchroot_install_files() {
+ sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
+ sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
# Install package builder script
sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
@@ -38,5 +45,4 @@ do_build() {
GROUP_ID=$(id -g)
sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID $GROUP_ID
- sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
}
diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
index 49aeb47..57d52ba 100644
--- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
+++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
@@ -12,10 +12,25 @@ SRC_URI = " \
file://configscript.sh \
file://README.sdk"
PV = "0.1"
-
-inherit isar-bootstrap-helper
PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
+TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
+TOOLCHAIN_${HOST_ARCH} = "build-essential"
+TOOLCHAIN_i386 = "build-essential"
+
+inherit rootfs
+ROOTFS_ARCH = "${HOST_ARCH}"
+ROOTFS_DISTRO = "${HOST_DISTRO}"
+ROOTFSDIR = "${S}"
+ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
+ROOTFS_FEATURES += "copy-package-cache"
+
+python() {
+ if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
+ raise bb.parse.SkipRecipe("SDK doesn't support {} as host".format(
+ d.getVar("ROOTFS_ARCH")))
+}
+
SDKCHROOT_PREINSTALL := "debhelper \
autotools-dev \
dpkg \
@@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
S = "${WORKDIR}/rootfs"
-do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_build[root_cleandirs] = "${S} \
- ${S}/isar-apt"
-
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-host:do_bootstrap"
-
-do_build() {
-
- if [ ${HOST_DISTRO} != "debian-stretch" ]; then
- bbfatal "SDK doesn't support ${HOST_DISTRO}"
- fi
- if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
- bbfatal "SDK doesn't support ${HOST_ARCH} as host"
- fi
-
- if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386" ]; then
- packages="${SDKCHROOT_PREINSTALL} build-essential"
- else
- packages="${SDKCHROOT_PREINSTALL} crossbuild-essential-${DISTRO_ARCH}"
- fi
-
- setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}' "${S}" $packages
+ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
+rootfs_configure_isar_apt_dir() {
+ # Copy isar-apt instead of mounting:
+ sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
+}
+ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
+sdkchroot_install_files() {
# Configure root filesystem
sudo install -m 644 ${WORKDIR}/README.sdk ${S}
sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-23 14:55 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline claudius.heine.ext
@ 2019-05-24 12:49 ` Maxim Yu. Osipov
2019-05-27 6:55 ` Claudius Heine
2019-08-14 16:00 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline Jan Kiszka
1 sibling, 1 reply; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-24 12:49 UTC (permalink / raw)
To: claudius.heine.ext, isar-users; +Cc: Claudius Heine
Hi Claudius,
One may agree that setup_root_file_system is too "fat"
and should be split to logical parts.
My concern is that one have to carefully keep track for every
ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of execution
- possible source of errors.
Just a case: one my not define weight value (in such case default value
is 20) as its is done for
ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if we
have several such default weights the order of commands could be wrong.
Regards,
Maxim.
On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> This removes the isar-bootstrap-helper.bbclass and integrates its
> functionality into the rootfs.bbclass.
>
> It introduces the rootfs_install task that runs through multiple
> functions from command lists in order to allow easy integration of
> custom functions.
>
> The introduced functions are, in general order of execution:
>
> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
> rootfs_configure_*: basic configuration of the root file system,
> before any custom packages where installed. (`/dev`, `/proc`, etc.
> are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
> rootfs_do_mount: mount standard directories and repositories into the
> rootfs
> do_rootfs_install_*: downloading and installing packages. Functions
> are added to `ROOTFS_INSTALL_COMMAND`.
>
> Signed-off-by: Claudius Heine <ch@denx.de>
> ---
> meta/classes/image.bbclass | 24 +--
> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
> .../isar-bootstrap/isar-bootstrap.inc | 5 +
> .../buildchroot/buildchroot-host.bb | 16 +-
> .../buildchroot/buildchroot-target.bb | 2 -
> .../buildchroot/buildchroot.inc | 26 ++-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
> 8 files changed, 228 insertions(+), 197 deletions(-)
> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index d77626c..99eea92 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -51,8 +51,10 @@ image_do_mounts() {
> buildchroot_do_mounts
> }
>
> -inherit isar-bootstrap-helper
> -ROOTFS_FEATURES += "finalize-rootfs"
> +ROOTFSDIR = "${IMAGE_ROOTFS}"
> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
> +
> inherit rootfs
> inherit image-sdk-extension
> inherit image-cache-extension
> @@ -119,8 +121,10 @@ python set_image_size () {
> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
> }
>
> -do_image_gen_fstab() {
> - cat > ${WORKDIR}/fstab << EOF
> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
> +image_configure_fstab[weight] = "2"
> +image_configure_fstab() {
> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
> # Begin /etc/fstab
> /dev/root / auto defaults 0 0
> proc /proc proc nosuid,noexec,nodev 0 0
> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs mode=0755,nosuid 0 0
> # End /etc/fstab
> EOF
> }
> -addtask image_gen_fstab before do_rootfs_install
> -
> -do_rootfs_install[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
> -do_rootfs_install[deptask] = "do_deploy_deb"
> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
> - ${IMAGE_ROOTFS}/isar-apt"
> -do_rootfs_install() {
> - setup_root_file_system --clean --keep-apt-cache \
> - --fstab "${WORKDIR}/fstab" \
> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
> -}
> -addtask rootfs_install before do_build after do_unpack
>
> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
> do_copy_boot_files() {
> diff --git a/meta/classes/isar-bootstrap-helper.bbclass b/meta/classes/isar-bootstrap-helper.bbclass
> deleted file mode 100644
> index bbc6322..0000000
> --- a/meta/classes/isar-bootstrap-helper.bbclass
> +++ /dev/null
> @@ -1,132 +0,0 @@
> -# Helper functions for using isar-bootstrap
> -#
> -# This software is a part of ISAR.
> -# Copyright (c) Siemens AG, 2018
> -#
> -# SPDX-License-Identifier: MIT
> -
> -def reverse_bb_array(d, varname):
> - array = d.getVar(varname, True)
> - if array is None:
> - return None
> - array = reversed(array.split())
> - return " ".join(i for i in array)
> -
> -
> -update_etc_os_release() {
> - OS_RELEASE_BUILD_ID=""
> - OS_RELEASE_VARIANT=""
> - while true; do
> - case "$1" in
> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
> - -*) bbfatal "$0: invalid option specified: $1" ;;
> - *) break ;;
> - esac
> - shift
> - done
> - ROOTFSDIR="$1"
> -
> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
> - sudo tee -a ${ROOTFSDIR}/etc/os-release
> - fi
> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
> - sudo tee -a ${ROOTFSDIR}/etc/os-release
> - fi
> -}
> -
> -setup_root_file_system() {
> - CLEAN=""
> - COPYISARAPT=""
> - FSTAB=""
> - ROOTFS_ARCH="${DISTRO_ARCH}"
> - ROOTFS_DISTRO="${DISTRO}"
> - while true; do
> - case "$1" in
> - --clean) CLEAN=1 ;;
> - --copyisarapt) COPYISARAPT=1 ;;
> - --fstab) FSTAB=$2; shift ;;
> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
> - -*) bbfatal "$0: invalid option specified: $1" ;;
> - *) break ;;
> - esac
> - shift
> - done
> - ROOTFSDIR="$1"
> - shift
> - PACKAGES="$@"
> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname ${ROOTFSDIR}/etc/resolv.conf"
> -
> - sudo cp -Trpfx \
> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
> - "$ROOTFSDIR"
> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee "$ROOTFSDIR/etc/fstab"
> -
> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main" | \
> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list" >/dev/null
> -
> - echo "Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
> -
> - if [ ${COPYISARAPT} ]; then
> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
> - else
> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
> - fi
> -
> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
> - fi
> -
> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
> - sudo mount --make-rslave ${ROOTFSDIR}/dev
> - sudo mount -t proc none $ROOTFSDIR/proc
> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
> - sudo mount --make-rslave ${ROOTFSDIR}/sys
> -
> - # Refresh /etc/resolv.conf
> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
> -
> - # Install packages:
> - E="${@ bb.utils.export_proxies(d)}"
> - export DEBIAN_FRONTEND=noninteractive
> - # To avoid Perl locale warnings:
> - export LANG=C
> - export LANGUAGE=C
> - export LC_ALL=C
> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
> - -o Dir::Etc::sourceparts="-" \
> - -o APT::Get::List-Cleanup="0"
> - # Add multiarch for cross-target
> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
> - fi
> - sudo -E chroot "$ROOTFSDIR" \
> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
> - sudo -E chroot "$ROOTFSDIR" \
> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
> - if [ ${CLEAN} ]; then
> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
> - mkdir -p ${WORKDIR}/apt_cache
> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t ${WORKDIR}/apt_cache '{}' '+'
> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
> - fi
> - sudo -E chroot "$ROOTFSDIR" \
> - /usr/bin/apt-get autoremove --purge --yes
> - sudo -E chroot "$ROOTFSDIR" \
> - /usr/bin/apt-get clean
> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh" "$ROOTFSDIR/var/lib/apt/lists/"*
> - fi
> -}
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 45b0350..0fc8ba8 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -1,14 +1,178 @@
> # This software is a part of ISAR.
> # Copyright (c) Siemens AG, 2019
>
> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
> +ROOTFS_DISTRO ?= "${DISTRO}"
> +ROOTFS_PACKAGES ?= ""
> +
> # Features of the rootfs creation:
> # available features are:
> +# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
> +# 'clean-package-cache' - delete package cache from rootfs
> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
> ROOTFS_FEATURES ?= ""
>
> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
> +
> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
> +
> +# Useful environment variables:
> +export E = "${@ bb.utils.export_proxies(d)}"
> +export DEBIAN_FRONTEND = "noninteractive"
> +# To avoid Perl locale warnings:
> +export LANG = "C"
> +export LANGUAGE = "C"
> +export LC_ALL = "C"
> +
> +rootfs_do_mounts[weight] = "3"
> +rootfs_do_mounts() {
> + sudo -s <<'EOSUDO'
> + mountpoint -q '${ROOTFSDIR}/dev' || \
> + mount --rbind /dev '${ROOTFSDIR}/dev'
> + mount --make-rslave '${ROOTFSDIR}/dev'
> + mountpoint -q '${ROOTFSDIR}/proc' || \
> + mount -t proc none '${ROOTFSDIR}/proc'
> + mountpoint -q '${ROOTFSDIR}/sys' || \
> + mount --rbind /sys '${ROOTFSDIR}/sys'
> + mount --make-rslave '${ROOTFSDIR}/sys'
> +
> + # Mount isar-apt if the directory does not exist or if it is empty
> + # This prevents overwriting something that was copied there
> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth 1 | wc -l)" = "0" ]
> + then
> + mkdir -p '${ROOTFSDIR}/isar-apt'
> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}' '${ROOTFSDIR}/isar-apt'
> + fi
> +
> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
> + if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}" = 'True' ]
> + then
> + mkdir -p '${ROOTFSDIR}/base-apt'
> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
> + mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
> + fi
> +
> +EOSUDO
> +}
> +
> +rootfs_do_qemu() {
> + if [ '${@repr(d.getVar('ROOTFS_ARCH') == d.getVar('HOST_ARCH'))}' = 'False' ]
> + then
> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static' '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
> + fi
> +}
> +
> +BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
> +BOOTSTRAP_SRC_${ROOTFS_ARCH} = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
> +
> +rootfs_prepare[weight] = "25"
> +rootfs_prepare(){
> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> +}
> +
> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
> +rootfs_configure_isar_apt[weight] = "2"
> +rootfs_configure_isar_apt() {
> + sudo -s <<'EOSUDO'
> +
> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
> +
> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
> +Package: *
> +Pin: release n=${DEBDISTRONAME}
> +Pin-Priority: 1000
> +EOF
> +EOSUDO
> +}
> +
> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
> +rootfs_install_pkgs_update[weight] = "5"
> +rootfs_install_pkgs_update() {
> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
> + -o Dir::Etc::sourceparts="-" \
> + -o APT::Get::List-Cleanup="0"
> +}
> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
> +rootfs_install_resolvconf[weight] = "1"
> +rootfs_install_resolvconf() {
> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
> +}
> +
> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
> +rootfs_install_pkgs_download[weight] = "600"
> +rootfs_install_pkgs_download() {
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
> +}
> +
> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
> +rootfs_install_clean_files[weight] = "2"
> +rootfs_install_clean_files() {
> + sudo -E chroot '${ROOTFSDIR}' \
> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
> +}
> +
> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
> +rootfs_install_pkgs_install[weight] = "8000"
> +rootfs_install_pkgs_install() {
> + sudo -E chroot "${ROOTFSDIR}" \
> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
> +}
Honestly I don't see benefits in such multiple
configure_cmds/rootfs_install_cmds - it doesn't increase code
readability as one have to additionally keep track for every command
'weight' value and it's easy to miss the correct order of commands
execution.
> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
> +do_rootfs_install[deptask] = "do_deploy_deb"
> +python do_rootfs_install() {
> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or "").split()
> +
> + # Mount after configure commands, so that they have time to copy
> + # 'isar-apt' (sdkchroot):
> + cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
> +
> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
> + for i in cmds]
> +
> + progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights)
> +
> + for cmd in cmds:
> + progress_reporter.next_stage()
> + bb.build.exec_func(cmd, d)
> + progress_reporter.finish()
> +}
> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
> +
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
> +rootfs_postprocess_copy_package_cache() {
> + mkdir -p '${WORKDIR}/apt_cache'
> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
> + me="$(id -u):$(id -g)"
> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
> +}
> +
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
> +rootfs_postprocess_clean_package_cache() {
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/bin/apt-get clean
> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
> +}
> +
> ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs', 'rootfs_postprocess_finalize', '', d)}"
> rootfs_postprocess_finalize() {
> sudo -s <<'EOSUDO'
> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
> +
> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
> find "${ROOTFSDIR}/usr/bin" \
> -maxdepth 1 -name 'qemu-*-static' -type f -delete
> @@ -42,6 +206,12 @@ EOSUDO
>
> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
> python do_rootfs_postprocess() {
> + # Take care that its correctly mounted:
> + bb.build.exec_func('rootfs_do_mounts', d)
> + # Take care that qemu-*-static is available, since it could have been
> + # removed on a previous execution of this task:
> + bb.build.exec_func('rootfs_do_qemu', d)
> +
> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
> if cmds is None or not cmds.strip():
> return
> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
> for cmd in cmds:
> bb.build.exec_func(cmd, d)
> }
> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
> +addtask rootfs_postprocess before do_rootfs
>
> python do_rootfs() {
> """Virtual task"""
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index b740c57..f8741c2 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -327,6 +327,11 @@ isar_bootstrap() {
> mount --make-rslave ${ROOTFSDIR}/sys
>
> export DEBIAN_FRONTEND=noninteractive
> +
> + if [ ${IS_HOST} ]; then
> + chroot "${ROOTFSDIR}" /usr/bin/dpkg --add-architecture ${DISTRO_ARCH}
> + fi
> +
> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
> -o Debug::pkgProblemResolver=yes
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> index 73cd548..121992c 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>
> require buildchroot.inc
> +ROOTFS_ARCH = "${HOST_ARCH}"
> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>
> BUILDCHROOT_PREINSTALL ?= "make \
> debhelper \
> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
> automake \
> devscripts \
> equivs \
> - libc6:${DISTRO_ARCH}"
> -
> -# According to the wiki page:
> -# https://wiki.debian.org/CrossToolchains
> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
> - crossbuild-essential-armhf"
> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
> - crossbuild-essential-arm64"
> -
> -
> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
> -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-host:do_bootstrap"
> + libc6:${DISTRO_ARCH} \
> + crossbuild-essential-${DISTRO_ARCH}"
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
> index 20f4d23..5a01258 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
> devscripts \
> equivs \
> adduser"
> -
> -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
> index ca9ce51..40fc8b0 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
> file://deps.sh"
> PV = "1.0"
>
> -inherit isar-bootstrap-helper
> +inherit rootfs
>
> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
> +ROOTFS_CLEAN_FILES = ""
> +
> +rootfs_do_mounts_append() {
> + sudo -s <<'EOSUDO'
> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
> +EOSUDO
> +}
>
> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
> - ${BUILDCHROOT_DIR}/isar-apt \
> - ${BUILDCHROOT_DIR}/downloads \
> - ${BUILDCHROOT_DIR}/home/builder"
> -
> -do_build() {
> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR} ${BUILDCHROOT_PREINSTALL}
> -
> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
> +buildchroot_install_files() {
> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
> # Install package builder script
> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
> @@ -38,5 +45,4 @@ do_build() {
> GROUP_ID=$(id -g)
> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID $GROUP_ID
>
> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
> }
> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> index 49aeb47..57d52ba 100644
> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
> @@ -12,10 +12,25 @@ SRC_URI = " \
> file://configscript.sh \
> file://README.sdk"
> PV = "0.1"
> -
> -inherit isar-bootstrap-helper
> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>
> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
> +TOOLCHAIN_i386 = "build-essential"
> +
> +inherit rootfs
> +ROOTFS_ARCH = "${HOST_ARCH}"
> +ROOTFS_DISTRO = "${HOST_DISTRO}"
> +ROOTFSDIR = "${S}"
> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
> +ROOTFS_FEATURES += "copy-package-cache"
> +
> +python() {
> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
> + raise bb.parse.SkipRecipe("SDK doesn't support {} as host".format(
> + d.getVar("ROOTFS_ARCH")))
> +}
> +
> SDKCHROOT_PREINSTALL := "debhelper \
> autotools-dev \
> dpkg \
> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>
> S = "${WORKDIR}/rootfs"
>
> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
> -do_build[root_cleandirs] = "${S} \
> - ${S}/isar-apt"
> -
> -do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-host:do_bootstrap"
> -
> -do_build() {
> -
> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
> - fi
> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
> - fi
> -
> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386" ]; then
> - packages="${SDKCHROOT_PREINSTALL} build-essential"
> - else
> - packages="${SDKCHROOT_PREINSTALL} crossbuild-essential-${DISTRO_ARCH}"
> - fi
> -
> - setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}' "${S}" $packages
> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
> +rootfs_configure_isar_apt_dir() {
> + # Copy isar-apt instead of mounting:
> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
> +}
>
> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
> +sdkchroot_install_files() {
> # Configure root filesystem
> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-24 12:49 ` Maxim Yu. Osipov
@ 2019-05-27 6:55 ` Claudius Heine
2019-05-27 7:20 ` Maxim Yu. Osipov
0 siblings, 1 reply; 30+ messages in thread
From: Claudius Heine @ 2019-05-27 6:55 UTC (permalink / raw)
To: Maxim Yu. Osipov, isar-users; +Cc: Claudius Heine
Hi Maxim,
On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
> Hi Claudius,
>
> One may agree that setup_root_file_system is too "fat"
> and should be split to logical parts.
There are multiple issues with `setup_root_file_system`, not just it
being 'fat'. For instance the amount of parameters this function accepts
just screams to me that it does not scale necessary customization.
> My concern is that one have to carefully keep track for every
> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of execution
> - possible source of errors.
The weight parameter does not specify the order or execution, but just
the percentage of the progress bar shown when building. Since
do_rootfs_install is a pretty big task, I found it useful to show at
least some progress.
The order of commands is the order in which they are listed in the
ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
Currently that is not an interface for the user, but that might happen
if more variables are added there to signify different stages of the
rootfs build process, for instance how the
`ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
regards,
Claudius
>
> Just a case: one my not define weight value (in such case default value
> is 20) as its is done for
> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if we
> have several such default weights the order of commands could be wrong.
>
> Regards,
> Maxim.
>
> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>> From: Claudius Heine <ch@denx.de>
>>
>> This removes the isar-bootstrap-helper.bbclass and integrates its
>> functionality into the rootfs.bbclass.
>>
>> It introduces the rootfs_install task that runs through multiple
>> functions from command lists in order to allow easy integration of
>> custom functions.
>>
>> The introduced functions are, in general order of execution:
>>
>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>> rootfs_configure_*: basic configuration of the root file system,
>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>> are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
>> rootfs_do_mount: mount standard directories and repositories into the
>> rootfs
>> do_rootfs_install_*: downloading and installing packages. Functions
>> are added to `ROOTFS_INSTALL_COMMAND`.
>>
>> Signed-off-by: Claudius Heine <ch@denx.de>
>> ---
>> meta/classes/image.bbclass | 24 +--
>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>> .../buildchroot/buildchroot-host.bb | 16 +-
>> .../buildchroot/buildchroot-target.bb | 2 -
>> .../buildchroot/buildchroot.inc | 26 ++-
>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>> 8 files changed, 228 insertions(+), 197 deletions(-)
>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>
>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>> index d77626c..99eea92 100644
>> --- a/meta/classes/image.bbclass
>> +++ b/meta/classes/image.bbclass
>> @@ -51,8 +51,10 @@ image_do_mounts() {
>> buildchroot_do_mounts
>> }
>> -inherit isar-bootstrap-helper
>> -ROOTFS_FEATURES += "finalize-rootfs"
>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>> finalize-rootfs"
>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>> +
>> inherit rootfs
>> inherit image-sdk-extension
>> inherit image-cache-extension
>> @@ -119,8 +121,10 @@ python set_image_size () {
>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>> }
>> -do_image_gen_fstab() {
>> - cat > ${WORKDIR}/fstab << EOF
>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>> +image_configure_fstab[weight] = "2"
>> +image_configure_fstab() {
>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>> # Begin /etc/fstab
>> /dev/root / auto defaults 0 0
>> proc /proc proc nosuid,noexec,nodev 0 0
>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>> mode=0755,nosuid 0 0
>> # End /etc/fstab
>> EOF
>> }
>> -addtask image_gen_fstab before do_rootfs_install
>> -
>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>> isar-bootstrap-target:do_bootstrap"
>> -do_rootfs_install[deptask] = "do_deploy_deb"
>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>> - ${IMAGE_ROOTFS}/isar-apt"
>> -do_rootfs_install() {
>> - setup_root_file_system --clean --keep-apt-cache \
>> - --fstab "${WORKDIR}/fstab" \
>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>> -}
>> -addtask rootfs_install before do_build after do_unpack
>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>> do_copy_boot_files() {
>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>> b/meta/classes/isar-bootstrap-helper.bbclass
>> deleted file mode 100644
>> index bbc6322..0000000
>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>> +++ /dev/null
>> @@ -1,132 +0,0 @@
>> -# Helper functions for using isar-bootstrap
>> -#
>> -# This software is a part of ISAR.
>> -# Copyright (c) Siemens AG, 2018
>> -#
>> -# SPDX-License-Identifier: MIT
>> -
>> -def reverse_bb_array(d, varname):
>> - array = d.getVar(varname, True)
>> - if array is None:
>> - return None
>> - array = reversed(array.split())
>> - return " ".join(i for i in array)
>> -
>> -
>> -update_etc_os_release() {
>> - OS_RELEASE_BUILD_ID=""
>> - OS_RELEASE_VARIANT=""
>> - while true; do
>> - case "$1" in
>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>> - *) break ;;
>> - esac
>> - shift
>> - done
>> - ROOTFSDIR="$1"
>> -
>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>> - fi
>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>> - fi
>> -}
>> -
>> -setup_root_file_system() {
>> - CLEAN=""
>> - COPYISARAPT=""
>> - FSTAB=""
>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>> - ROOTFS_DISTRO="${DISTRO}"
>> - while true; do
>> - case "$1" in
>> - --clean) CLEAN=1 ;;
>> - --copyisarapt) COPYISARAPT=1 ;;
>> - --fstab) FSTAB=$2; shift ;;
>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>> - *) break ;;
>> - esac
>> - shift
>> - done
>> - ROOTFSDIR="$1"
>> - shift
>> - PACKAGES="$@"
>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname ${ROOTFSDIR}/etc/resolv.conf"
>> -
>> - sudo cp -Trpfx \
>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>> - "$ROOTFSDIR"
>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee "$ROOTFSDIR/etc/fstab"
>> -
>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main" | \
>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>> >/dev/null
>> -
>> - echo "Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority:
>> 1000" | \
>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>> -
>> - if [ ${COPYISARAPT} ]; then
>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>> - else
>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>> - fi
>> -
>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>> - fi
>> -
>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>> - sudo mount -t proc none $ROOTFSDIR/proc
>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>> -
>> - # Refresh /etc/resolv.conf
>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>> -
>> - # Install packages:
>> - E="${@ bb.utils.export_proxies(d)}"
>> - export DEBIAN_FRONTEND=noninteractive
>> - # To avoid Perl locale warnings:
>> - export LANG=C
>> - export LANGUAGE=C
>> - export LC_ALL=C
>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>> - -o Dir::Etc::sourceparts="-" \
>> - -o APT::Get::List-Cleanup="0"
>> - # Add multiarch for cross-target
>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg --add-architecture
>> ${DISTRO_ARCH}
>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>> - fi
>> - sudo -E chroot "$ROOTFSDIR" \
>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>> - sudo -E chroot "$ROOTFSDIR" \
>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>> - if [ ${CLEAN} ]; then
>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>> - mkdir -p ${WORKDIR}/apt_cache
>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>> ${WORKDIR}/apt_cache '{}' '+'
>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>> - fi
>> - sudo -E chroot "$ROOTFSDIR" \
>> - /usr/bin/apt-get autoremove --purge --yes
>> - sudo -E chroot "$ROOTFSDIR" \
>> - /usr/bin/apt-get clean
>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>> "$ROOTFSDIR/var/lib/apt/lists/"*
>> - fi
>> -}
>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>> index 45b0350..0fc8ba8 100644
>> --- a/meta/classes/rootfs.bbclass
>> +++ b/meta/classes/rootfs.bbclass
>> @@ -1,14 +1,178 @@
>> # This software is a part of ISAR.
>> # Copyright (c) Siemens AG, 2019
>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>> +ROOTFS_DISTRO ?= "${DISTRO}"
>> +ROOTFS_PACKAGES ?= ""
>> +
>> # Features of the rootfs creation:
>> # available features are:
>> +# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
>> +# 'clean-package-cache' - delete package cache from rootfs
>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>> ROOTFS_FEATURES ?= ""
>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>> +
>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>> +
>> +# Useful environment variables:
>> +export E = "${@ bb.utils.export_proxies(d)}"
>> +export DEBIAN_FRONTEND = "noninteractive"
>> +# To avoid Perl locale warnings:
>> +export LANG = "C"
>> +export LANGUAGE = "C"
>> +export LC_ALL = "C"
>> +
>> +rootfs_do_mounts[weight] = "3"
>> +rootfs_do_mounts() {
>> + sudo -s <<'EOSUDO'
>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>> + mount --make-rslave '${ROOTFSDIR}/dev'
>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>> + mount -t proc none '${ROOTFSDIR}/proc'
>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>> + mount --make-rslave '${ROOTFSDIR}/sys'
>> +
>> + # Mount isar-apt if the directory does not exist or if it is
>> empty
>> + # This prevents overwriting something that was copied there
>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth 1
>> | wc -l)" = "0" ]
>> + then
>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>> '${ROOTFSDIR}/isar-apt'
>> + fi
>> +
>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>> + if [
>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}"
>> = 'True' ]
>> + then
>> + mkdir -p '${ROOTFSDIR}/base-apt'
>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>> + mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
>> + fi
>> +
>> +EOSUDO
>> +}
>> +
>> +rootfs_do_qemu() {
>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') == d.getVar('HOST_ARCH'))}'
>> = 'False' ]
>> + then
>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>> + fi
>> +}
>> +
>> +BOOTSTRAP_SRC =
>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>> +
>> +rootfs_prepare[weight] = "25"
>> +rootfs_prepare(){
>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>> +}
>> +
>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>> +rootfs_configure_isar_apt[weight] = "2"
>> +rootfs_configure_isar_apt() {
>> + sudo -s <<'EOSUDO'
>> +
>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>> +
>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>> +Package: *
>> +Pin: release n=${DEBDISTRONAME}
>> +Pin-Priority: 1000
>> +EOF
>> +EOSUDO
>> +}
>> +
>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>> +rootfs_install_pkgs_update[weight] = "5"
>> +rootfs_install_pkgs_update() {
>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>> + -o Dir::Etc::sourceparts="-" \
>> + -o APT::Get::List-Cleanup="0"
>> +}
>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>> +rootfs_install_resolvconf[weight] = "1"
>> +rootfs_install_resolvconf() {
>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>> +}
>> +
>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>> +rootfs_install_pkgs_download[weight] = "600"
>> +rootfs_install_pkgs_download() {
>> + sudo -E chroot '${ROOTFSDIR}' \
>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>> ${ROOTFS_PACKAGES}
>> +}
>> +
>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>> +rootfs_install_clean_files[weight] = "2"
>> +rootfs_install_clean_files() {
>> + sudo -E chroot '${ROOTFSDIR}' \
>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>> +}
>> +
>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>> +rootfs_install_pkgs_install[weight] = "8000"
>> +rootfs_install_pkgs_install() {
>> + sudo -E chroot "${ROOTFSDIR}" \
>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>> +}
>
>
> Honestly I don't see benefits in such multiple
> configure_cmds/rootfs_install_cmds - it doesn't increase code
> readability as one have to additionally keep track for every command
> 'weight' value and it's easy to miss the correct order of commands
> execution.
>
>
>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>> ${ROOTFS_INSTALL_COMMAND}"
>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>> 'host'}:do_build isar-apt:do_cache_config"
>> +do_rootfs_install[deptask] = "do_deploy_deb"
>> +python do_rootfs_install() {
>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or
>> "").split()
>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>> "").split()
>> +
>> + # Mount after configure commands, so that they have time to copy
>> + # 'isar-apt' (sdkchroot):
>> + cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts']
>> + install_cmds
>> +
>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>> + for i in cmds]
>> +
>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>> stage_weights)
>> +
>> + for cmd in cmds:
>> + progress_reporter.next_stage()
>> + bb.build.exec_func(cmd, d)
>> + progress_reporter.finish()
>> +}
>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>> +
>> +ROOTFS_POSTPROCESS_COMMAND +=
>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>> +rootfs_postprocess_copy_package_cache() {
>> + mkdir -p '${WORKDIR}/apt_cache'
>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>> '${WORKDIR}/apt_cache' '{}' '+'
>> + me="$(id -u):$(id -g)"
>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>> +}
>> +
>> +ROOTFS_POSTPROCESS_COMMAND +=
>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>> +rootfs_postprocess_clean_package_cache() {
>> + sudo -E chroot '${ROOTFSDIR}' \
>> + /usr/bin/apt-get clean
>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>> +}
>> +
>> ROOTFS_POSTPROCESS_COMMAND +=
>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>> 'rootfs_postprocess_finalize', '', d)}"
>> rootfs_postprocess_finalize() {
>> sudo -s <<'EOSUDO'
>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>> +
>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>> find "${ROOTFSDIR}/usr/bin" \
>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>> @@ -42,6 +206,12 @@ EOSUDO
>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>> python do_rootfs_postprocess() {
>> + # Take care that its correctly mounted:
>> + bb.build.exec_func('rootfs_do_mounts', d)
>> + # Take care that qemu-*-static is available, since it could have
>> been
>> + # removed on a previous execution of this task:
>> + bb.build.exec_func('rootfs_do_qemu', d)
>> +
>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>> if cmds is None or not cmds.strip():
>> return
>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>> for cmd in cmds:
>> bb.build.exec_func(cmd, d)
>> }
>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>> +addtask rootfs_postprocess before do_rootfs
>> python do_rootfs() {
>> """Virtual task"""
>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> index b740c57..f8741c2 100644
>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>> mount --make-rslave ${ROOTFSDIR}/sys
>> export DEBIAN_FRONTEND=noninteractive
>> +
>> + if [ ${IS_HOST} ]; then
>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>> --add-architecture ${DISTRO_ARCH}
>> + fi
>> +
>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>> -o Debug::pkgProblemResolver=yes
>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> index 73cd548..121992c 100644
>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>> require buildchroot.inc
>> +ROOTFS_ARCH = "${HOST_ARCH}"
>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>> BUILDCHROOT_PREINSTALL ?= "make \
>> debhelper \
>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>> automake \
>> devscripts \
>> equivs \
>> - libc6:${DISTRO_ARCH}"
>> -
>> -# According to the wiki page:
>> -# https://wiki.debian.org/CrossToolchains
>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>> - crossbuild-essential-armhf"
>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>> - crossbuild-essential-arm64"
>> -
>> -
>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>> -do_build[depends] = "isar-apt:do_cache_config
>> isar-bootstrap-host:do_bootstrap"
>> + libc6:${DISTRO_ARCH} \
>> + crossbuild-essential-${DISTRO_ARCH}"
>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>> index 20f4d23..5a01258 100644
>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>> devscripts \
>> equivs \
>> adduser"
>> -
>> -do_build[depends] = "isar-apt:do_cache_config
>> isar-bootstrap-target:do_bootstrap"
>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>> index ca9ce51..40fc8b0 100644
>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>> file://deps.sh"
>> PV = "1.0"
>> -inherit isar-bootstrap-helper
>> +inherit rootfs
>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>> +ROOTFS_CLEAN_FILES = ""
>> +
>> +rootfs_do_mounts_append() {
>> + sudo -s <<'EOSUDO'
>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>> +EOSUDO
>> +}
>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>> - ${BUILDCHROOT_DIR}/isar-apt \
>> - ${BUILDCHROOT_DIR}/downloads \
>> - ${BUILDCHROOT_DIR}/home/builder"
>> -
>> -do_build() {
>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>> ${BUILDCHROOT_PREINSTALL}
>> -
>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>> +buildchroot_install_files() {
>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>> # Install package builder script
>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>> @@ -38,5 +45,4 @@ do_build() {
>> GROUP_ID=$(id -g)
>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID $GROUP_ID
>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>> }
>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>> index 49aeb47..57d52ba 100644
>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>> @@ -12,10 +12,25 @@ SRC_URI = " \
>> file://configscript.sh \
>> file://README.sdk"
>> PV = "0.1"
>> -
>> -inherit isar-bootstrap-helper
>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>> +TOOLCHAIN_i386 = "build-essential"
>> +
>> +inherit rootfs
>> +ROOTFS_ARCH = "${HOST_ARCH}"
>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>> +ROOTFSDIR = "${S}"
>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>> +ROOTFS_FEATURES += "copy-package-cache"
>> +
>> +python() {
>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>> host".format(
>> + d.getVar("ROOTFS_ARCH")))
>> +}
>> +
>> SDKCHROOT_PREINSTALL := "debhelper \
>> autotools-dev \
>> dpkg \
>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>> S = "${WORKDIR}/rootfs"
>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>> -do_build[root_cleandirs] = "${S} \
>> - ${S}/isar-apt"
>> -
>> -do_build[depends] = "isar-apt:do_cache_config
>> isar-bootstrap-host:do_bootstrap"
>> -
>> -do_build() {
>> -
>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>> - fi
>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>> - fi
>> -
>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386" ];
>> then
>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>> - else
>> - packages="${SDKCHROOT_PREINSTALL}
>> crossbuild-essential-${DISTRO_ARCH}"
>> - fi
>> -
>> - setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}'
>> --host-distro '${HOST_DISTRO}' "${S}" $packages
>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>> +rootfs_configure_isar_apt_dir() {
>> + # Copy isar-apt instead of mounting:
>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>> +}
>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>> +sdkchroot_install_files() {
>> # Configure root filesystem
>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>
>
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-27 6:55 ` Claudius Heine
@ 2019-05-27 7:20 ` Maxim Yu. Osipov
2019-05-27 7:36 ` Maxim Yu. Osipov
2019-05-27 9:03 ` Claudius Heine
0 siblings, 2 replies; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 7:20 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
Hi Claudius,
On 5/27/19 8:55 AM, Claudius Heine wrote:
> Hi Maxim,
>
> On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
>> Hi Claudius,
>>
>> One may agree that setup_root_file_system is too "fat"
>> and should be split to logical parts.
>
> There are multiple issues with `setup_root_file_system`, not just it
> being 'fat'. For instance the amount of parameters this function accepts
> just screams to me that it does not scale necessary customization.
>
>> My concern is that one have to carefully keep track for every
>> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of
>> execution - possible source of errors.
>
> The weight parameter does not specify the order or execution, but just
> the percentage of the progress bar shown when building. Since
> do_rootfs_install is a pretty big task, I found it useful to show at
> least some progress.
Ah, I see now. Please add this comment into the code (to avoid confusion).
>
> The order of commands is the order in which they are listed in the
> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
>
> Currently that is not an interface for the user, but that might happen
> if more variables are added there to signify different stages of the
> rootfs build process, for instance how the
> `ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
I think that it's worth to add description of
ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND in RECIPE_API_CHANGELOG.
Regards,
Maxim.
> regards,
> Claudius
>
>>
>> Just a case: one my not define weight value (in such case default
>> value is 20) as its is done for
>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if we
>> have several such default weights the order of commands could be wrong.
>>
>> Regards,
>> Maxim.
>>
>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>> From: Claudius Heine <ch@denx.de>
>>>
>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>> functionality into the rootfs.bbclass.
>>>
>>> It introduces the rootfs_install task that runs through multiple
>>> functions from command lists in order to allow easy integration of
>>> custom functions.
>>>
>>> The introduced functions are, in general order of execution:
>>>
>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>> rootfs_configure_*: basic configuration of the root file system,
>>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>>> are not mounted). Functions are added to
>>> `ROOTFS_CONFIGURE_COMMAND`.
>>> rootfs_do_mount: mount standard directories and repositories into the
>>> rootfs
>>> do_rootfs_install_*: downloading and installing packages. Functions
>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>
>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>> ---
>>> meta/classes/image.bbclass | 24 +--
>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>> .../buildchroot/buildchroot-target.bb | 2 -
>>> .../buildchroot/buildchroot.inc | 26 ++-
>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>
>>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>>> index d77626c..99eea92 100644
>>> --- a/meta/classes/image.bbclass
>>> +++ b/meta/classes/image.bbclass
>>> @@ -51,8 +51,10 @@ image_do_mounts() {
>>> buildchroot_do_mounts
>>> }
>>> -inherit isar-bootstrap-helper
>>> -ROOTFS_FEATURES += "finalize-rootfs"
>>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>>> finalize-rootfs"
>>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>> +
>>> inherit rootfs
>>> inherit image-sdk-extension
>>> inherit image-cache-extension
>>> @@ -119,8 +121,10 @@ python set_image_size () {
>>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>>> }
>>> -do_image_gen_fstab() {
>>> - cat > ${WORKDIR}/fstab << EOF
>>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>>> +image_configure_fstab[weight] = "2"
>>> +image_configure_fstab() {
>>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>>> # Begin /etc/fstab
>>> /dev/root / auto defaults 0 0
>>> proc /proc proc nosuid,noexec,nodev 0 0
>>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>>> mode=0755,nosuid 0 0
>>> # End /etc/fstab
>>> EOF
>>> }
>>> -addtask image_gen_fstab before do_rootfs_install
>>> -
>>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>>> isar-bootstrap-target:do_bootstrap"
>>> -do_rootfs_install[deptask] = "do_deploy_deb"
>>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>>> - ${IMAGE_ROOTFS}/isar-apt"
>>> -do_rootfs_install() {
>>> - setup_root_file_system --clean --keep-apt-cache \
>>> - --fstab "${WORKDIR}/fstab" \
>>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>>> -}
>>> -addtask rootfs_install before do_build after do_unpack
>>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>>> do_copy_boot_files() {
>>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>>> b/meta/classes/isar-bootstrap-helper.bbclass
>>> deleted file mode 100644
>>> index bbc6322..0000000
>>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>>> +++ /dev/null
>>> @@ -1,132 +0,0 @@
>>> -# Helper functions for using isar-bootstrap
>>> -#
>>> -# This software is a part of ISAR.
>>> -# Copyright (c) Siemens AG, 2018
>>> -#
>>> -# SPDX-License-Identifier: MIT
>>> -
>>> -def reverse_bb_array(d, varname):
>>> - array = d.getVar(varname, True)
>>> - if array is None:
>>> - return None
>>> - array = reversed(array.split())
>>> - return " ".join(i for i in array)
>>> -
>>> -
>>> -update_etc_os_release() {
>>> - OS_RELEASE_BUILD_ID=""
>>> - OS_RELEASE_VARIANT=""
>>> - while true; do
>>> - case "$1" in
>>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>> - *) break ;;
>>> - esac
>>> - shift
>>> - done
>>> - ROOTFSDIR="$1"
>>> -
>>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>> - fi
>>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>> - fi
>>> -}
>>> -
>>> -setup_root_file_system() {
>>> - CLEAN=""
>>> - COPYISARAPT=""
>>> - FSTAB=""
>>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>>> - ROOTFS_DISTRO="${DISTRO}"
>>> - while true; do
>>> - case "$1" in
>>> - --clean) CLEAN=1 ;;
>>> - --copyisarapt) COPYISARAPT=1 ;;
>>> - --fstab) FSTAB=$2; shift ;;
>>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>> - *) break ;;
>>> - esac
>>> - shift
>>> - done
>>> - ROOTFSDIR="$1"
>>> - shift
>>> - PACKAGES="$@"
>>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname
>>> ${ROOTFSDIR}/etc/resolv.conf"
>>> -
>>> - sudo cp -Trpfx \
>>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>>> - "$ROOTFSDIR"
>>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee "$ROOTFSDIR/etc/fstab"
>>> -
>>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main" | \
>>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>>> >/dev/null
>>> -
>>> - echo "Package: *\nPin: release n=${DEBDISTRONAME}\nPin-Priority:
>>> 1000" | \
>>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>>> -
>>> - if [ ${COPYISARAPT} ]; then
>>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>>> - else
>>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO}
>>> $ROOTFSDIR/isar-apt
>>> - fi
>>> -
>>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>>> - fi
>>> -
>>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>>> - sudo mount -t proc none $ROOTFSDIR/proc
>>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>>> -
>>> - # Refresh /etc/resolv.conf
>>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>>> -
>>> - # Install packages:
>>> - E="${@ bb.utils.export_proxies(d)}"
>>> - export DEBIAN_FRONTEND=noninteractive
>>> - # To avoid Perl locale warnings:
>>> - export LANG=C
>>> - export LANGUAGE=C
>>> - export LC_ALL=C
>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>> - -o Dir::Etc::sourceparts="-" \
>>> - -o APT::Get::List-Cleanup="0"
>>> - # Add multiarch for cross-target
>>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg --add-architecture
>>> ${DISTRO_ARCH}
>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>>> - fi
>>> - sudo -E chroot "$ROOTFSDIR" \
>>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>>> - sudo -E chroot "$ROOTFSDIR" \
>>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>>> - if [ ${CLEAN} ]; then
>>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>>> - mkdir -p ${WORKDIR}/apt_cache
>>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>> ${WORKDIR}/apt_cache '{}' '+'
>>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>>> - fi
>>> - sudo -E chroot "$ROOTFSDIR" \
>>> - /usr/bin/apt-get autoremove --purge --yes
>>> - sudo -E chroot "$ROOTFSDIR" \
>>> - /usr/bin/apt-get clean
>>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>>> "$ROOTFSDIR/var/lib/apt/lists/"*
>>> - fi
>>> -}
>>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>>> index 45b0350..0fc8ba8 100644
>>> --- a/meta/classes/rootfs.bbclass
>>> +++ b/meta/classes/rootfs.bbclass
>>> @@ -1,14 +1,178 @@
>>> # This software is a part of ISAR.
>>> # Copyright (c) Siemens AG, 2019
>>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>>> +ROOTFS_DISTRO ?= "${DISTRO}"
>>> +ROOTFS_PACKAGES ?= ""
>>> +
>>> # Features of the rootfs creation:
>>> # available features are:
>>> +# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
>>> +# 'clean-package-cache' - delete package cache from rootfs
>>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>>> ROOTFS_FEATURES ?= ""
>>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>> +
>>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>>> +
>>> +# Useful environment variables:
>>> +export E = "${@ bb.utils.export_proxies(d)}"
>>> +export DEBIAN_FRONTEND = "noninteractive"
>>> +# To avoid Perl locale warnings:
>>> +export LANG = "C"
>>> +export LANGUAGE = "C"
>>> +export LC_ALL = "C"
>>> +
>>> +rootfs_do_mounts[weight] = "3"
>>> +rootfs_do_mounts() {
>>> + sudo -s <<'EOSUDO'
>>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>>> + mount --make-rslave '${ROOTFSDIR}/dev'
>>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>>> + mount -t proc none '${ROOTFSDIR}/proc'
>>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>>> + mount --make-rslave '${ROOTFSDIR}/sys'
>>> +
>>> + # Mount isar-apt if the directory does not exist or if it is
>>> empty
>>> + # This prevents overwriting something that was copied there
>>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth 1
>>> | wc -l)" = "0" ]
>>> + then
>>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>>> '${ROOTFSDIR}/isar-apt'
>>> + fi
>>> +
>>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>>> + if [
>>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}" =
>>> 'True' ]
>>> + then
>>> + mkdir -p '${ROOTFSDIR}/base-apt'
>>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>>> + mount --bind '${REPO_BASE_DIR}' '${ROOTFSDIR}/base-apt'
>>> + fi
>>> +
>>> +EOSUDO
>>> +}
>>> +
>>> +rootfs_do_qemu() {
>>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') ==
>>> d.getVar('HOST_ARCH'))}' = 'False' ]
>>> + then
>>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>>> + fi
>>> +}
>>> +
>>> +BOOTSTRAP_SRC =
>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>>> +
>>> +rootfs_prepare[weight] = "25"
>>> +rootfs_prepare(){
>>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>>> +}
>>> +
>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>>> +rootfs_configure_isar_apt[weight] = "2"
>>> +rootfs_configure_isar_apt() {
>>> + sudo -s <<'EOSUDO'
>>> +
>>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
>>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>>> +
>>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>>> +Package: *
>>> +Pin: release n=${DEBDISTRONAME}
>>> +Pin-Priority: 1000
>>> +EOF
>>> +EOSUDO
>>> +}
>>> +
>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>>> +rootfs_install_pkgs_update[weight] = "5"
>>> +rootfs_install_pkgs_update() {
>>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>> + -o Dir::Etc::sourceparts="-" \
>>> + -o APT::Get::List-Cleanup="0"
>>> +}
>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>>> +rootfs_install_resolvconf[weight] = "1"
>>> +rootfs_install_resolvconf() {
>>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>>> +}
>>> +
>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>>> +rootfs_install_pkgs_download[weight] = "600"
>>> +rootfs_install_pkgs_download() {
>>> + sudo -E chroot '${ROOTFSDIR}' \
>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>>> ${ROOTFS_PACKAGES}
>>> +}
>>> +
>>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>>> +rootfs_install_clean_files[weight] = "2"
>>> +rootfs_install_clean_files() {
>>> + sudo -E chroot '${ROOTFSDIR}' \
>>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>>> +}
>>> +
>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>>> +rootfs_install_pkgs_install[weight] = "8000"
>>> +rootfs_install_pkgs_install() {
>>> + sudo -E chroot "${ROOTFSDIR}" \
>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>>> +}
>>
>>
>> Honestly I don't see benefits in such multiple
>> configure_cmds/rootfs_install_cmds - it doesn't increase code
>> readability as one have to additionally keep track for every command
>> 'weight' value and it's easy to miss the correct order of commands
>> execution.
>>
>>
>>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>>> ${ROOTFS_INSTALL_COMMAND}"
>>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>>> 'host'}:do_build isar-apt:do_cache_config"
>>> +do_rootfs_install[deptask] = "do_deploy_deb"
>>> +python do_rootfs_install() {
>>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or
>>> "").split()
>>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>>> "").split()
>>> +
>>> + # Mount after configure commands, so that they have time to copy
>>> + # 'isar-apt' (sdkchroot):
>>> + cmds = ['rootfs_prepare'] + configure_cmds +
>>> ['rootfs_do_mounts'] + install_cmds
>>> +
>>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>>> + for i in cmds]
>>> +
>>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>>> stage_weights)
>>> +
>>> + for cmd in cmds:
>>> + progress_reporter.next_stage()
>>> + bb.build.exec_func(cmd, d)
>>> + progress_reporter.finish()
>>> +}
>>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>>> +
>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>>> +rootfs_postprocess_copy_package_cache() {
>>> + mkdir -p '${WORKDIR}/apt_cache'
>>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>> '${WORKDIR}/apt_cache' '{}' '+'
>>> + me="$(id -u):$(id -g)"
>>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>>> +}
>>> +
>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>>> +rootfs_postprocess_clean_package_cache() {
>>> + sudo -E chroot '${ROOTFSDIR}' \
>>> + /usr/bin/apt-get clean
>>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>>> +}
>>> +
>>> ROOTFS_POSTPROCESS_COMMAND +=
>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>>> 'rootfs_postprocess_finalize', '', d)}"
>>> rootfs_postprocess_finalize() {
>>> sudo -s <<'EOSUDO'
>>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>>> +
>>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>>> find "${ROOTFSDIR}/usr/bin" \
>>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>>> @@ -42,6 +206,12 @@ EOSUDO
>>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>>> python do_rootfs_postprocess() {
>>> + # Take care that its correctly mounted:
>>> + bb.build.exec_func('rootfs_do_mounts', d)
>>> + # Take care that qemu-*-static is available, since it could have
>>> been
>>> + # removed on a previous execution of this task:
>>> + bb.build.exec_func('rootfs_do_qemu', d)
>>> +
>>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>>> if cmds is None or not cmds.strip():
>>> return
>>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>>> for cmd in cmds:
>>> bb.build.exec_func(cmd, d)
>>> }
>>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>>> +addtask rootfs_postprocess before do_rootfs
>>> python do_rootfs() {
>>> """Virtual task"""
>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> index b740c57..f8741c2 100644
>>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>>> mount --make-rslave ${ROOTFSDIR}/sys
>>> export DEBIAN_FRONTEND=noninteractive
>>> +
>>> + if [ ${IS_HOST} ]; then
>>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>>> --add-architecture ${DISTRO_ARCH}
>>> + fi
>>> +
>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>> -o Debug::pkgProblemResolver=yes
>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> index 73cd548..121992c 100644
>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>> require buildchroot.inc
>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>> BUILDCHROOT_PREINSTALL ?= "make \
>>> debhelper \
>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>> automake \
>>> devscripts \
>>> equivs \
>>> - libc6:${DISTRO_ARCH}"
>>> -
>>> -# According to the wiki page:
>>> -# https://wiki.debian.org/CrossToolchains
>>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>>> - crossbuild-essential-armhf"
>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>> - crossbuild-essential-arm64"
>>> -
>>> -
>>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>>> -do_build[depends] = "isar-apt:do_cache_config
>>> isar-bootstrap-host:do_bootstrap"
>>> + libc6:${DISTRO_ARCH} \
>>> + crossbuild-essential-${DISTRO_ARCH}"
>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>> index 20f4d23..5a01258 100644
>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>>> devscripts \
>>> equivs \
>>> adduser"
>>> -
>>> -do_build[depends] = "isar-apt:do_cache_config
>>> isar-bootstrap-target:do_bootstrap"
>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>> index ca9ce51..40fc8b0 100644
>>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>>> file://deps.sh"
>>> PV = "1.0"
>>> -inherit isar-bootstrap-helper
>>> +inherit rootfs
>>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>>> +ROOTFS_CLEAN_FILES = ""
>>> +
>>> +rootfs_do_mounts_append() {
>>> + sudo -s <<'EOSUDO'
>>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>>> +EOSUDO
>>> +}
>>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>>> - ${BUILDCHROOT_DIR}/isar-apt \
>>> - ${BUILDCHROOT_DIR}/downloads \
>>> - ${BUILDCHROOT_DIR}/home/builder"
>>> -
>>> -do_build() {
>>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>>> ${BUILDCHROOT_PREINSTALL}
>>> -
>>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>>> +buildchroot_install_files() {
>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>>> # Install package builder script
>>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>>> @@ -38,5 +45,4 @@ do_build() {
>>> GROUP_ID=$(id -g)
>>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID $GROUP_ID
>>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>>> }
>>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>> index 49aeb47..57d52ba 100644
>>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>> @@ -12,10 +12,25 @@ SRC_URI = " \
>>> file://configscript.sh \
>>> file://README.sdk"
>>> PV = "0.1"
>>> -
>>> -inherit isar-bootstrap-helper
>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>>> +TOOLCHAIN_i386 = "build-essential"
>>> +
>>> +inherit rootfs
>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>> +ROOTFSDIR = "${S}"
>>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>>> +ROOTFS_FEATURES += "copy-package-cache"
>>> +
>>> +python() {
>>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>>> host".format(
>>> + d.getVar("ROOTFS_ARCH")))
>>> +}
>>> +
>>> SDKCHROOT_PREINSTALL := "debhelper \
>>> autotools-dev \
>>> dpkg \
>>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>>> S = "${WORKDIR}/rootfs"
>>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>>> -do_build[root_cleandirs] = "${S} \
>>> - ${S}/isar-apt"
>>> -
>>> -do_build[depends] = "isar-apt:do_cache_config
>>> isar-bootstrap-host:do_bootstrap"
>>> -
>>> -do_build() {
>>> -
>>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>>> - fi
>>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>>> - fi
>>> -
>>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386" ];
>>> then
>>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>>> - else
>>> - packages="${SDKCHROOT_PREINSTALL}
>>> crossbuild-essential-${DISTRO_ARCH}"
>>> - fi
>>> -
>>> - setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}'
>>> --host-distro '${HOST_DISTRO}' "${S}" $packages
>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>>> +rootfs_configure_isar_apt_dir() {
>>> + # Copy isar-apt instead of mounting:
>>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>>> +}
>>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>>> +sdkchroot_install_files() {
>>> # Configure root filesystem
>>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>>
>>
>>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-27 7:20 ` Maxim Yu. Osipov
@ 2019-05-27 7:36 ` Maxim Yu. Osipov
2019-05-27 8:30 ` Claudius Heine
2019-05-27 9:03 ` Claudius Heine
1 sibling, 1 reply; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 7:36 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
Hi again Claudius,
I think that my question in first email is still valid:
Honestly I don't see benefits in such multiple
configure_cmds/rootfs_install_cmds in rootfs.class -
it doesn't increase code readability (as for me I was confused regarding
order of execution).
Why not collapse them into the single rootfs_install_* function?
Regards,
Maxim.
On 5/27/19 9:20 AM, Maxim Yu. Osipov wrote:
> Hi Claudius,
>
> On 5/27/19 8:55 AM, Claudius Heine wrote:
>> Hi Maxim,
>>
>> On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
>>> Hi Claudius,
>>>
>>> One may agree that setup_root_file_system is too "fat"
>>> and should be split to logical parts.
>>
>> There are multiple issues with `setup_root_file_system`, not just it
>> being 'fat'. For instance the amount of parameters this function
>> accepts just screams to me that it does not scale necessary
>> customization.
>>
>>> My concern is that one have to carefully keep track for every
>>> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of
>>> execution - possible source of errors.
>>
>> The weight parameter does not specify the order or execution, but just
>> the percentage of the progress bar shown when building. Since
>> do_rootfs_install is a pretty big task, I found it useful to show at
>> least some progress.
>
> Ah, I see now. Please add this comment into the code (to avoid confusion).
>
>>
>> The order of commands is the order in which they are listed in the
>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
>>
>> Currently that is not an interface for the user, but that might happen
>> if more variables are added there to signify different stages of the
>> rootfs build process, for instance how the
>> `ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
>
> I think that it's worth to add description of
> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND in RECIPE_API_CHANGELOG.
>
> Regards,
> Maxim.
>
>
>> regards,
>> Claudius
>>
>>>
>>> Just a case: one my not define weight value (in such case default
>>> value is 20) as its is done for
>>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if
>>> we have several such default weights the order of commands could be
>>> wrong.
>>>
>>> Regards,
>>> Maxim.
>>>
>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>> From: Claudius Heine <ch@denx.de>
>>>>
>>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>>> functionality into the rootfs.bbclass.
>>>>
>>>> It introduces the rootfs_install task that runs through multiple
>>>> functions from command lists in order to allow easy integration of
>>>> custom functions.
>>>>
>>>> The introduced functions are, in general order of execution:
>>>>
>>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>>> rootfs_configure_*: basic configuration of the root file system,
>>>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>>>> are not mounted). Functions are added to
>>>> `ROOTFS_CONFIGURE_COMMAND`.
>>>> rootfs_do_mount: mount standard directories and repositories into
>>>> the
>>>> rootfs
>>>> do_rootfs_install_*: downloading and installing packages. Functions
>>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>>
>>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>>> ---
>>>> meta/classes/image.bbclass | 24 +--
>>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>>> meta/classes/rootfs.bbclass | 172
>>>> +++++++++++++++++-
>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>> .../buildchroot/buildchroot.inc | 26 ++-
>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>
>>>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>>>> index d77626c..99eea92 100644
>>>> --- a/meta/classes/image.bbclass
>>>> +++ b/meta/classes/image.bbclass
>>>> @@ -51,8 +51,10 @@ image_do_mounts() {
>>>> buildchroot_do_mounts
>>>> }
>>>> -inherit isar-bootstrap-helper
>>>> -ROOTFS_FEATURES += "finalize-rootfs"
>>>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>>>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>>>> finalize-rootfs"
>>>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>>> +
>>>> inherit rootfs
>>>> inherit image-sdk-extension
>>>> inherit image-cache-extension
>>>> @@ -119,8 +121,10 @@ python set_image_size () {
>>>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>>>> }
>>>> -do_image_gen_fstab() {
>>>> - cat > ${WORKDIR}/fstab << EOF
>>>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>>>> +image_configure_fstab[weight] = "2"
>>>> +image_configure_fstab() {
>>>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>>>> # Begin /etc/fstab
>>>> /dev/root / auto defaults 0 0
>>>> proc /proc proc nosuid,noexec,nodev 0 0
>>>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>>>> mode=0755,nosuid 0 0
>>>> # End /etc/fstab
>>>> EOF
>>>> }
>>>> -addtask image_gen_fstab before do_rootfs_install
>>>> -
>>>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-target:do_bootstrap"
>>>> -do_rootfs_install[deptask] = "do_deploy_deb"
>>>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>>>> - ${IMAGE_ROOTFS}/isar-apt"
>>>> -do_rootfs_install() {
>>>> - setup_root_file_system --clean --keep-apt-cache \
>>>> - --fstab "${WORKDIR}/fstab" \
>>>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>>>> -}
>>>> -addtask rootfs_install before do_build after do_unpack
>>>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>> do_copy_boot_files() {
>>>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>>>> b/meta/classes/isar-bootstrap-helper.bbclass
>>>> deleted file mode 100644
>>>> index bbc6322..0000000
>>>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>>>> +++ /dev/null
>>>> @@ -1,132 +0,0 @@
>>>> -# Helper functions for using isar-bootstrap
>>>> -#
>>>> -# This software is a part of ISAR.
>>>> -# Copyright (c) Siemens AG, 2018
>>>> -#
>>>> -# SPDX-License-Identifier: MIT
>>>> -
>>>> -def reverse_bb_array(d, varname):
>>>> - array = d.getVar(varname, True)
>>>> - if array is None:
>>>> - return None
>>>> - array = reversed(array.split())
>>>> - return " ".join(i for i in array)
>>>> -
>>>> -
>>>> -update_etc_os_release() {
>>>> - OS_RELEASE_BUILD_ID=""
>>>> - OS_RELEASE_VARIANT=""
>>>> - while true; do
>>>> - case "$1" in
>>>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>>>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>> - *) break ;;
>>>> - esac
>>>> - shift
>>>> - done
>>>> - ROOTFSDIR="$1"
>>>> -
>>>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>>>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>>>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>> - fi
>>>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>>>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>>>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>> - fi
>>>> -}
>>>> -
>>>> -setup_root_file_system() {
>>>> - CLEAN=""
>>>> - COPYISARAPT=""
>>>> - FSTAB=""
>>>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>>>> - ROOTFS_DISTRO="${DISTRO}"
>>>> - while true; do
>>>> - case "$1" in
>>>> - --clean) CLEAN=1 ;;
>>>> - --copyisarapt) COPYISARAPT=1 ;;
>>>> - --fstab) FSTAB=$2; shift ;;
>>>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>>>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>>>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>> - *) break ;;
>>>> - esac
>>>> - shift
>>>> - done
>>>> - ROOTFSDIR="$1"
>>>> - shift
>>>> - PACKAGES="$@"
>>>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname
>>>> ${ROOTFSDIR}/etc/resolv.conf"
>>>> -
>>>> - sudo cp -Trpfx \
>>>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>>>> - "$ROOTFSDIR"
>>>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee
>>>> "$ROOTFSDIR/etc/fstab"
>>>> -
>>>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main"
>>>> | \
>>>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>>>> >/dev/null
>>>> -
>>>> - echo "Package: *\nPin: release
>>>> n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
>>>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>>>> -
>>>> - if [ ${COPYISARAPT} ]; then
>>>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>>>> - else
>>>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO}
>>>> $ROOTFSDIR/isar-apt
>>>> - fi
>>>> -
>>>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>>>> - fi
>>>> -
>>>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>>>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>>>> - sudo mount -t proc none $ROOTFSDIR/proc
>>>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>>>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>>>> -
>>>> - # Refresh /etc/resolv.conf
>>>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>>>> -
>>>> - # Install packages:
>>>> - E="${@ bb.utils.export_proxies(d)}"
>>>> - export DEBIAN_FRONTEND=noninteractive
>>>> - # To avoid Perl locale warnings:
>>>> - export LANG=C
>>>> - export LANGUAGE=C
>>>> - export LC_ALL=C
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>>>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>> - -o Dir::Etc::sourceparts="-" \
>>>> - -o APT::Get::List-Cleanup="0"
>>>> - # Add multiarch for cross-target
>>>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg
>>>> --add-architecture ${DISTRO_ARCH}
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>>>> - fi
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>>>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>>>> - if [ ${CLEAN} ]; then
>>>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>>>> - mkdir -p ${WORKDIR}/apt_cache
>>>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>>>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>> ${WORKDIR}/apt_cache '{}' '+'
>>>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>>>> - fi
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get autoremove --purge --yes
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get clean
>>>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>>>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>>>> "$ROOTFSDIR/var/lib/apt/lists/"*
>>>> - fi
>>>> -}
>>>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>>>> index 45b0350..0fc8ba8 100644
>>>> --- a/meta/classes/rootfs.bbclass
>>>> +++ b/meta/classes/rootfs.bbclass
>>>> @@ -1,14 +1,178 @@
>>>> # This software is a part of ISAR.
>>>> # Copyright (c) Siemens AG, 2019
>>>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>>>> +ROOTFS_DISTRO ?= "${DISTRO}"
>>>> +ROOTFS_PACKAGES ?= ""
>>>> +
>>>> # Features of the rootfs creation:
>>>> # available features are:
>>>> +# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
>>>> +# 'clean-package-cache' - delete package cache from rootfs
>>>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>>>> ROOTFS_FEATURES ?= ""
>>>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>> +
>>>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>>>> +
>>>> +# Useful environment variables:
>>>> +export E = "${@ bb.utils.export_proxies(d)}"
>>>> +export DEBIAN_FRONTEND = "noninteractive"
>>>> +# To avoid Perl locale warnings:
>>>> +export LANG = "C"
>>>> +export LANGUAGE = "C"
>>>> +export LC_ALL = "C"
>>>> +
>>>> +rootfs_do_mounts[weight] = "3"
>>>> +rootfs_do_mounts() {
>>>> + sudo -s <<'EOSUDO'
>>>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>>>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>>>> + mount --make-rslave '${ROOTFSDIR}/dev'
>>>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>>>> + mount -t proc none '${ROOTFSDIR}/proc'
>>>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>>>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>>>> + mount --make-rslave '${ROOTFSDIR}/sys'
>>>> +
>>>> + # Mount isar-apt if the directory does not exist or if it
>>>> is empty
>>>> + # This prevents overwriting something that was copied there
>>>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>>>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth
>>>> 1 | wc -l)" = "0" ]
>>>> + then
>>>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>>>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>>>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>>>> '${ROOTFSDIR}/isar-apt'
>>>> + fi
>>>> +
>>>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>>>> + if [
>>>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}"
>>>> = 'True' ]
>>>> + then
>>>> + mkdir -p '${ROOTFSDIR}/base-apt'
>>>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>>>> + mount --bind '${REPO_BASE_DIR}'
>>>> '${ROOTFSDIR}/base-apt'
>>>> + fi
>>>> +
>>>> +EOSUDO
>>>> +}
>>>> +
>>>> +rootfs_do_qemu() {
>>>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') ==
>>>> d.getVar('HOST_ARCH'))}' = 'False' ]
>>>> + then
>>>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>>>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>>>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>>>> + fi
>>>> +}
>>>> +
>>>> +BOOTSTRAP_SRC =
>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>>>>
>>>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>>>> +
>>>> +rootfs_prepare[weight] = "25"
>>>> +rootfs_prepare(){
>>>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>>>> +}
>>>> +
>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>>>> +rootfs_configure_isar_apt[weight] = "2"
>>>> +rootfs_configure_isar_apt() {
>>>> + sudo -s <<'EOSUDO'
>>>> +
>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>>>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main'
>>>> > \
>>>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>>>> +
>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>>>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>>>> +Package: *
>>>> +Pin: release n=${DEBDISTRONAME}
>>>> +Pin-Priority: 1000
>>>> +EOF
>>>> +EOSUDO
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>>>> +rootfs_install_pkgs_update[weight] = "5"
>>>> +rootfs_install_pkgs_update() {
>>>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>>>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>> + -o Dir::Etc::sourceparts="-" \
>>>> + -o APT::Get::List-Cleanup="0"
>>>> +}
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>>>> +rootfs_install_resolvconf[weight] = "1"
>>>> +rootfs_install_resolvconf() {
>>>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>>>> +rootfs_install_pkgs_download[weight] = "600"
>>>> +rootfs_install_pkgs_download() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>>>> ${ROOTFS_PACKAGES}
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>>>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>>>> +rootfs_install_clean_files[weight] = "2"
>>>> +rootfs_install_clean_files() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>>>> +rootfs_install_pkgs_install[weight] = "8000"
>>>> +rootfs_install_pkgs_install() {
>>>> + sudo -E chroot "${ROOTFSDIR}" \
>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>>>> +}
>>>
>>>
>>> Honestly I don't see benefits in such multiple
>>> configure_cmds/rootfs_install_cmds - it doesn't increase code
>>> readability as one have to additionally keep track for every command
>>> 'weight' value and it's easy to miss the correct order of commands
>>> execution.
>>>
>>>
>>>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>>>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>>>> ${ROOTFS_INSTALL_COMMAND}"
>>>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>>>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>>>> 'host'}:do_build isar-apt:do_cache_config"
>>>> +do_rootfs_install[deptask] = "do_deploy_deb"
>>>> +python do_rootfs_install() {
>>>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or
>>>> "").split()
>>>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>>>> "").split()
>>>> +
>>>> + # Mount after configure commands, so that they have time to copy
>>>> + # 'isar-apt' (sdkchroot):
>>>> + cmds = ['rootfs_prepare'] + configure_cmds +
>>>> ['rootfs_do_mounts'] + install_cmds
>>>> +
>>>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>>>> + for i in cmds]
>>>> +
>>>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>>>> stage_weights)
>>>> +
>>>> + for cmd in cmds:
>>>> + progress_reporter.next_stage()
>>>> + bb.build.exec_func(cmd, d)
>>>> + progress_reporter.finish()
>>>> +}
>>>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>>>> +
>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>>>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>>>> +rootfs_postprocess_copy_package_cache() {
>>>> + mkdir -p '${WORKDIR}/apt_cache'
>>>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>>>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>> '${WORKDIR}/apt_cache' '{}' '+'
>>>> + me="$(id -u):$(id -g)"
>>>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>>>> +}
>>>> +
>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>>>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>>>> +rootfs_postprocess_clean_package_cache() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /usr/bin/apt-get clean
>>>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>>>> +}
>>>> +
>>>> ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>>>> 'rootfs_postprocess_finalize', '', d)}"
>>>> rootfs_postprocess_finalize() {
>>>> sudo -s <<'EOSUDO'
>>>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>>>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>>>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>>>> +
>>>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>>>> find "${ROOTFSDIR}/usr/bin" \
>>>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>>>> @@ -42,6 +206,12 @@ EOSUDO
>>>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>>>> python do_rootfs_postprocess() {
>>>> + # Take care that its correctly mounted:
>>>> + bb.build.exec_func('rootfs_do_mounts', d)
>>>> + # Take care that qemu-*-static is available, since it could
>>>> have been
>>>> + # removed on a previous execution of this task:
>>>> + bb.build.exec_func('rootfs_do_qemu', d)
>>>> +
>>>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>>>> if cmds is None or not cmds.strip():
>>>> return
>>>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>>>> for cmd in cmds:
>>>> bb.build.exec_func(cmd, d)
>>>> }
>>>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>>>> +addtask rootfs_postprocess before do_rootfs
>>>> python do_rootfs() {
>>>> """Virtual task"""
>>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> index b740c57..f8741c2 100644
>>>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>>>> mount --make-rslave ${ROOTFSDIR}/sys
>>>> export DEBIAN_FRONTEND=noninteractive
>>>> +
>>>> + if [ ${IS_HOST} ]; then
>>>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>>>> --add-architecture ${DISTRO_ARCH}
>>>> + fi
>>>> +
>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>> -o Debug::pkgProblemResolver=yes
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> index 73cd548..121992c 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>> require buildchroot.inc
>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>> BUILDCHROOT_PREINSTALL ?= "make \
>>>> debhelper \
>>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>>> automake \
>>>> devscripts \
>>>> equivs \
>>>> - libc6:${DISTRO_ARCH}"
>>>> -
>>>> -# According to the wiki page:
>>>> -# https://wiki.debian.org/CrossToolchains
>>>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>>>> - crossbuild-essential-armhf"
>>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>>> - crossbuild-essential-arm64"
>>>> -
>>>> -
>>>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-host:do_bootstrap"
>>>> + libc6:${DISTRO_ARCH} \
>>>> + crossbuild-essential-${DISTRO_ARCH}"
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> index 20f4d23..5a01258 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>>>> devscripts \
>>>> equivs \
>>>> adduser"
>>>> -
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-target:do_bootstrap"
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> index ca9ce51..40fc8b0 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>>>> file://deps.sh"
>>>> PV = "1.0"
>>>> -inherit isar-bootstrap-helper
>>>> +inherit rootfs
>>>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>>>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>>>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>>>> +ROOTFS_CLEAN_FILES = ""
>>>> +
>>>> +rootfs_do_mounts_append() {
>>>> + sudo -s <<'EOSUDO'
>>>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>>>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>>>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>>>> +EOSUDO
>>>> +}
>>>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>>>> - ${BUILDCHROOT_DIR}/isar-apt \
>>>> - ${BUILDCHROOT_DIR}/downloads \
>>>> - ${BUILDCHROOT_DIR}/home/builder"
>>>> -
>>>> -do_build() {
>>>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>>>> ${BUILDCHROOT_PREINSTALL}
>>>> -
>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>>>> +buildchroot_install_files() {
>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>>>> # Install package builder script
>>>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>>>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>>>> @@ -38,5 +45,4 @@ do_build() {
>>>> GROUP_ID=$(id -g)
>>>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID
>>>> $GROUP_ID
>>>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>>>> }
>>>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> index 49aeb47..57d52ba 100644
>>>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> @@ -12,10 +12,25 @@ SRC_URI = " \
>>>> file://configscript.sh \
>>>> file://README.sdk"
>>>> PV = "0.1"
>>>> -
>>>> -inherit isar-bootstrap-helper
>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>>>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>>>> +TOOLCHAIN_i386 = "build-essential"
>>>> +
>>>> +inherit rootfs
>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>> +ROOTFSDIR = "${S}"
>>>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>>>> +ROOTFS_FEATURES += "copy-package-cache"
>>>> +
>>>> +python() {
>>>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>>>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>>>> host".format(
>>>> + d.getVar("ROOTFS_ARCH")))
>>>> +}
>>>> +
>>>> SDKCHROOT_PREINSTALL := "debhelper \
>>>> autotools-dev \
>>>> dpkg \
>>>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>>>> S = "${WORKDIR}/rootfs"
>>>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>> -do_build[root_cleandirs] = "${S} \
>>>> - ${S}/isar-apt"
>>>> -
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-host:do_bootstrap"
>>>> -
>>>> -do_build() {
>>>> -
>>>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>>>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>>>> - fi
>>>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>>>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>>>> - fi
>>>> -
>>>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386"
>>>> ]; then
>>>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>>>> - else
>>>> - packages="${SDKCHROOT_PREINSTALL}
>>>> crossbuild-essential-${DISTRO_ARCH}"
>>>> - fi
>>>> -
>>>> - setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}'
>>>> --host-distro '${HOST_DISTRO}' "${S}" $packages
>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>>>> +rootfs_configure_isar_apt_dir() {
>>>> + # Copy isar-apt instead of mounting:
>>>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>>>> +}
>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>>>> +sdkchroot_install_files() {
>>>> # Configure root filesystem
>>>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>>>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>>>
>>>
>>>
>>
>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-27 7:36 ` Maxim Yu. Osipov
@ 2019-05-27 8:30 ` Claudius Heine
0 siblings, 0 replies; 30+ messages in thread
From: Claudius Heine @ 2019-05-27 8:30 UTC (permalink / raw)
To: Maxim Yu. Osipov, isar-users; +Cc: Claudius Heine
On 27/05/2019 09.36, Maxim Yu. Osipov wrote:
> Hi again Claudius,
>
> I think that my question in first email is still valid:
>
> Honestly I don't see benefits in such multiple
> configure_cmds/rootfs_install_cmds in rootfs.class -
> it doesn't increase code readability (as for me I was confused regarding
> order of execution).
> Why not collapse them into the single rootfs_install_* function?
Well, it is a bit late to have that argument. The right point would have
been when I posted the RFC commit over a month ago or better over two
months ago when I described my design the first time.
It also increase code readability, because the `setup_root_file_system`
had so many special cases and parameters that it is very difficult to
figure out what is exactly happening and why. And it is not much more
difficult than figuring out in which order bitbake tasks are executed.
It might be even easier, since those task have a deterministic order and
can be easily inspected using 'bitbake -e ...'.
The number of parameters and special cases of `setup_root_file_system`
also shows pretty clearly that there is a need to add custom code for
new features or for downstream usage in between certain steps of the
root file system creation.
Just putting those in multiple tasks is not possible, because they would
not be idempotent.
This design was inspired by openembedded [1]. So I do expect that it is
battle tested.
[1]
https://git.openembedded.org/openembedded-core/tree/meta/classes/rootfs-postcommands.bbclass
>
> Regards,
> Maxim.
>
> On 5/27/19 9:20 AM, Maxim Yu. Osipov wrote:
>> Hi Claudius,
>>
>> On 5/27/19 8:55 AM, Claudius Heine wrote:
>>> Hi Maxim,
>>>
>>> On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
>>>> Hi Claudius,
>>>>
>>>> One may agree that setup_root_file_system is too "fat"
>>>> and should be split to logical parts.
>>>
>>> There are multiple issues with `setup_root_file_system`, not just it
>>> being 'fat'. For instance the amount of parameters this function
>>> accepts just screams to me that it does not scale necessary
>>> customization.
>>>
>>>> My concern is that one have to carefully keep track for every
>>>> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of
>>>> execution - possible source of errors.
>>>
>>> The weight parameter does not specify the order or execution, but
>>> just the percentage of the progress bar shown when building. Since
>>> do_rootfs_install is a pretty big task, I found it useful to show at
>>> least some progress.
>>
>> Ah, I see now. Please add this comment into the code (to avoid
>> confusion).
>>
>>>
>>> The order of commands is the order in which they are listed in the
>>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
>>>
>>> Currently that is not an interface for the user, but that might
>>> happen if more variables are added there to signify different stages
>>> of the rootfs build process, for instance how the
>>> `ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
>>
>> I think that it's worth to add description of
>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND in RECIPE_API_CHANGELOG.
>>
>> Regards,
>> Maxim.
>>
>>
>>> regards,
>>> Claudius
>>>
>>>>
>>>> Just a case: one my not define weight value (in such case default
>>>> value is 20) as its is done for
>>>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if
>>>> we have several such default weights the order of commands could be
>>>> wrong.
>>>>
>>>> Regards,
>>>> Maxim.
>>>>
>>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>>> From: Claudius Heine <ch@denx.de>
>>>>>
>>>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>>>> functionality into the rootfs.bbclass.
>>>>>
>>>>> It introduces the rootfs_install task that runs through multiple
>>>>> functions from command lists in order to allow easy integration of
>>>>> custom functions.
>>>>>
>>>>> The introduced functions are, in general order of execution:
>>>>>
>>>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>>>> rootfs_configure_*: basic configuration of the root file system,
>>>>> before any custom packages where installed. (`/dev`, `/proc`,
>>>>> etc.
>>>>> are not mounted). Functions are added to
>>>>> `ROOTFS_CONFIGURE_COMMAND`.
>>>>> rootfs_do_mount: mount standard directories and repositories
>>>>> into the
>>>>> rootfs
>>>>> do_rootfs_install_*: downloading and installing packages. Functions
>>>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>>>
>>>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>>>> ---
>>>>> meta/classes/image.bbclass | 24 +--
>>>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>>>> meta/classes/rootfs.bbclass | 172
>>>>> +++++++++++++++++-
>>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>>> .../buildchroot/buildchroot.inc | 26 ++-
>>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>>
>>>>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>>>>> index d77626c..99eea92 100644
>>>>> --- a/meta/classes/image.bbclass
>>>>> +++ b/meta/classes/image.bbclass
>>>>> @@ -51,8 +51,10 @@ image_do_mounts() {
>>>>> buildchroot_do_mounts
>>>>> }
>>>>> -inherit isar-bootstrap-helper
>>>>> -ROOTFS_FEATURES += "finalize-rootfs"
>>>>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>>>>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>>>>> finalize-rootfs"
>>>>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>>>> +
>>>>> inherit rootfs
>>>>> inherit image-sdk-extension
>>>>> inherit image-cache-extension
>>>>> @@ -119,8 +121,10 @@ python set_image_size () {
>>>>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>>>>> }
>>>>> -do_image_gen_fstab() {
>>>>> - cat > ${WORKDIR}/fstab << EOF
>>>>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>>>>> +image_configure_fstab[weight] = "2"
>>>>> +image_configure_fstab() {
>>>>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>>>>> # Begin /etc/fstab
>>>>> /dev/root / auto defaults 0 0
>>>>> proc /proc proc nosuid,noexec,nodev 0 0
>>>>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>>>>> mode=0755,nosuid 0 0
>>>>> # End /etc/fstab
>>>>> EOF
>>>>> }
>>>>> -addtask image_gen_fstab before do_rootfs_install
>>>>> -
>>>>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-target:do_bootstrap"
>>>>> -do_rootfs_install[deptask] = "do_deploy_deb"
>>>>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>>>>> - ${IMAGE_ROOTFS}/isar-apt"
>>>>> -do_rootfs_install() {
>>>>> - setup_root_file_system --clean --keep-apt-cache \
>>>>> - --fstab "${WORKDIR}/fstab" \
>>>>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>>>>> -}
>>>>> -addtask rootfs_install before do_build after do_unpack
>>>>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>>> do_copy_boot_files() {
>>>>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>>>>> b/meta/classes/isar-bootstrap-helper.bbclass
>>>>> deleted file mode 100644
>>>>> index bbc6322..0000000
>>>>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>>>>> +++ /dev/null
>>>>> @@ -1,132 +0,0 @@
>>>>> -# Helper functions for using isar-bootstrap
>>>>> -#
>>>>> -# This software is a part of ISAR.
>>>>> -# Copyright (c) Siemens AG, 2018
>>>>> -#
>>>>> -# SPDX-License-Identifier: MIT
>>>>> -
>>>>> -def reverse_bb_array(d, varname):
>>>>> - array = d.getVar(varname, True)
>>>>> - if array is None:
>>>>> - return None
>>>>> - array = reversed(array.split())
>>>>> - return " ".join(i for i in array)
>>>>> -
>>>>> -
>>>>> -update_etc_os_release() {
>>>>> - OS_RELEASE_BUILD_ID=""
>>>>> - OS_RELEASE_VARIANT=""
>>>>> - while true; do
>>>>> - case "$1" in
>>>>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>>>>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>>> - *) break ;;
>>>>> - esac
>>>>> - shift
>>>>> - done
>>>>> - ROOTFSDIR="$1"
>>>>> -
>>>>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>>>>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>>>>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>>> - fi
>>>>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>>>>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>>>>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>>> - fi
>>>>> -}
>>>>> -
>>>>> -setup_root_file_system() {
>>>>> - CLEAN=""
>>>>> - COPYISARAPT=""
>>>>> - FSTAB=""
>>>>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>>>>> - ROOTFS_DISTRO="${DISTRO}"
>>>>> - while true; do
>>>>> - case "$1" in
>>>>> - --clean) CLEAN=1 ;;
>>>>> - --copyisarapt) COPYISARAPT=1 ;;
>>>>> - --fstab) FSTAB=$2; shift ;;
>>>>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>>>>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>>>>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>>> - *) break ;;
>>>>> - esac
>>>>> - shift
>>>>> - done
>>>>> - ROOTFSDIR="$1"
>>>>> - shift
>>>>> - PACKAGES="$@"
>>>>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname
>>>>> ${ROOTFSDIR}/etc/resolv.conf"
>>>>> -
>>>>> - sudo cp -Trpfx \
>>>>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>>>>> - "$ROOTFSDIR"
>>>>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee
>>>>> "$ROOTFSDIR/etc/fstab"
>>>>> -
>>>>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME}
>>>>> main" | \
>>>>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>>>>> >/dev/null
>>>>> -
>>>>> - echo "Package: *\nPin: release
>>>>> n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
>>>>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>>>>> -
>>>>> - if [ ${COPYISARAPT} ]; then
>>>>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>>>>> - else
>>>>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO}
>>>>> $ROOTFSDIR/isar-apt
>>>>> - fi
>>>>> -
>>>>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>>>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>>>>> - fi
>>>>> -
>>>>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>>>>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>>>>> - sudo mount -t proc none $ROOTFSDIR/proc
>>>>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>>>>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>>>>> -
>>>>> - # Refresh /etc/resolv.conf
>>>>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>>>>> -
>>>>> - # Install packages:
>>>>> - E="${@ bb.utils.export_proxies(d)}"
>>>>> - export DEBIAN_FRONTEND=noninteractive
>>>>> - # To avoid Perl locale warnings:
>>>>> - export LANG=C
>>>>> - export LANGUAGE=C
>>>>> - export LC_ALL=C
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>>>>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>>> - -o Dir::Etc::sourceparts="-" \
>>>>> - -o APT::Get::List-Cleanup="0"
>>>>> - # Add multiarch for cross-target
>>>>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg
>>>>> --add-architecture ${DISTRO_ARCH}
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>>>>> - fi
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>>>>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>>>>> - if [ ${CLEAN} ]; then
>>>>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>>>>> - mkdir -p ${WORKDIR}/apt_cache
>>>>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>>>>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>>> ${WORKDIR}/apt_cache '{}' '+'
>>>>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>>>>> - fi
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get autoremove --purge --yes
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get clean
>>>>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>>>>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>>>>> "$ROOTFSDIR/var/lib/apt/lists/"*
>>>>> - fi
>>>>> -}
>>>>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>>>>> index 45b0350..0fc8ba8 100644
>>>>> --- a/meta/classes/rootfs.bbclass
>>>>> +++ b/meta/classes/rootfs.bbclass
>>>>> @@ -1,14 +1,178 @@
>>>>> # This software is a part of ISAR.
>>>>> # Copyright (c) Siemens AG, 2019
>>>>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>>>>> +ROOTFS_DISTRO ?= "${DISTRO}"
>>>>> +ROOTFS_PACKAGES ?= ""
>>>>> +
>>>>> # Features of the rootfs creation:
>>>>> # available features are:
>>>>> +# 'deploy-package-cache' - copy the package cache
>>>>> ${WORKDIR}/apt_cache
>>>>> +# 'clean-package-cache' - delete package cache from rootfs
>>>>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>>>>> ROOTFS_FEATURES ?= ""
>>>>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>>> +
>>>>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>>>>> +
>>>>> +# Useful environment variables:
>>>>> +export E = "${@ bb.utils.export_proxies(d)}"
>>>>> +export DEBIAN_FRONTEND = "noninteractive"
>>>>> +# To avoid Perl locale warnings:
>>>>> +export LANG = "C"
>>>>> +export LANGUAGE = "C"
>>>>> +export LC_ALL = "C"
>>>>> +
>>>>> +rootfs_do_mounts[weight] = "3"
>>>>> +rootfs_do_mounts() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>>>>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>>>>> + mount --make-rslave '${ROOTFSDIR}/dev'
>>>>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>>>>> + mount -t proc none '${ROOTFSDIR}/proc'
>>>>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>>>>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>>>>> + mount --make-rslave '${ROOTFSDIR}/sys'
>>>>> +
>>>>> + # Mount isar-apt if the directory does not exist or if it
>>>>> is empty
>>>>> + # This prevents overwriting something that was copied there
>>>>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>>>>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth
>>>>> 1 | wc -l)" = "0" ]
>>>>> + then
>>>>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>>>>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>>>>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>>>>> '${ROOTFSDIR}/isar-apt'
>>>>> + fi
>>>>> +
>>>>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>>>>> + if [
>>>>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}"
>>>>> = 'True' ]
>>>>> + then
>>>>> + mkdir -p '${ROOTFSDIR}/base-apt'
>>>>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>>>>> + mount --bind '${REPO_BASE_DIR}'
>>>>> '${ROOTFSDIR}/base-apt'
>>>>> + fi
>>>>> +
>>>>> +EOSUDO
>>>>> +}
>>>>> +
>>>>> +rootfs_do_qemu() {
>>>>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') ==
>>>>> d.getVar('HOST_ARCH'))}' = 'False' ]
>>>>> + then
>>>>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>>>>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>>>>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>>>>> + fi
>>>>> +}
>>>>> +
>>>>> +BOOTSTRAP_SRC =
>>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>>>>>
>>>>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>>>>> +
>>>>> +rootfs_prepare[weight] = "25"
>>>>> +rootfs_prepare(){
>>>>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>>>>> +rootfs_configure_isar_apt[weight] = "2"
>>>>> +rootfs_configure_isar_apt() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> +
>>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>>>>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME}
>>>>> main' > \
>>>>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>>>>> +
>>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>>>>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>>>>> +Package: *
>>>>> +Pin: release n=${DEBDISTRONAME}
>>>>> +Pin-Priority: 1000
>>>>> +EOF
>>>>> +EOSUDO
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>>>>> +rootfs_install_pkgs_update[weight] = "5"
>>>>> +rootfs_install_pkgs_update() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>>>>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>>> + -o Dir::Etc::sourceparts="-" \
>>>>> + -o APT::Get::List-Cleanup="0"
>>>>> +}
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>>>>> +rootfs_install_resolvconf[weight] = "1"
>>>>> +rootfs_install_resolvconf() {
>>>>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>>>>> +rootfs_install_pkgs_download[weight] = "600"
>>>>> +rootfs_install_pkgs_download() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>>>>> ${ROOTFS_PACKAGES}
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>>>>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>>>>> +rootfs_install_clean_files[weight] = "2"
>>>>> +rootfs_install_clean_files() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>>>>> +rootfs_install_pkgs_install[weight] = "8000"
>>>>> +rootfs_install_pkgs_install() {
>>>>> + sudo -E chroot "${ROOTFSDIR}" \
>>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>>>>> +}
>>>>
>>>>
>>>> Honestly I don't see benefits in such multiple
>>>> configure_cmds/rootfs_install_cmds - it doesn't increase code
>>>> readability as one have to additionally keep track for every command
>>>> 'weight' value and it's easy to miss the correct order of commands
>>>> execution.
>>>>
>>>>
>>>>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>>>>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>>>>> ${ROOTFS_INSTALL_COMMAND}"
>>>>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>>>>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>>>>> 'host'}:do_build isar-apt:do_cache_config"
>>>>> +do_rootfs_install[deptask] = "do_deploy_deb"
>>>>> +python do_rootfs_install() {
>>>>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True)
>>>>> or "").split()
>>>>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>>>>> "").split()
>>>>> +
>>>>> + # Mount after configure commands, so that they have time to copy
>>>>> + # 'isar-apt' (sdkchroot):
>>>>> + cmds = ['rootfs_prepare'] + configure_cmds +
>>>>> ['rootfs_do_mounts'] + install_cmds
>>>>> +
>>>>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>>>>> + for i in cmds]
>>>>> +
>>>>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>>>>> stage_weights)
>>>>> +
>>>>> + for cmd in cmds:
>>>>> + progress_reporter.next_stage()
>>>>> + bb.build.exec_func(cmd, d)
>>>>> + progress_reporter.finish()
>>>>> +}
>>>>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>>>>> +
>>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>>>>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>>>>> +rootfs_postprocess_copy_package_cache() {
>>>>> + mkdir -p '${WORKDIR}/apt_cache'
>>>>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>>>>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>>> '${WORKDIR}/apt_cache' '{}' '+'
>>>>> + me="$(id -u):$(id -g)"
>>>>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>>>>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>>>>> +rootfs_postprocess_clean_package_cache() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /usr/bin/apt-get clean
>>>>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>>>>> +}
>>>>> +
>>>>> ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>>>>> 'rootfs_postprocess_finalize', '', d)}"
>>>>> rootfs_postprocess_finalize() {
>>>>> sudo -s <<'EOSUDO'
>>>>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>>>>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>>>>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>>>>> +
>>>>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>>>>> find "${ROOTFSDIR}/usr/bin" \
>>>>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>>>>> @@ -42,6 +206,12 @@ EOSUDO
>>>>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>>>>> python do_rootfs_postprocess() {
>>>>> + # Take care that its correctly mounted:
>>>>> + bb.build.exec_func('rootfs_do_mounts', d)
>>>>> + # Take care that qemu-*-static is available, since it could
>>>>> have been
>>>>> + # removed on a previous execution of this task:
>>>>> + bb.build.exec_func('rootfs_do_qemu', d)
>>>>> +
>>>>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>>>>> if cmds is None or not cmds.strip():
>>>>> return
>>>>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>>>>> for cmd in cmds:
>>>>> bb.build.exec_func(cmd, d)
>>>>> }
>>>>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>>>>> +addtask rootfs_postprocess before do_rootfs
>>>>> python do_rootfs() {
>>>>> """Virtual task"""
>>>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> index b740c57..f8741c2 100644
>>>>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>>>>> mount --make-rslave ${ROOTFSDIR}/sys
>>>>> export DEBIAN_FRONTEND=noninteractive
>>>>> +
>>>>> + if [ ${IS_HOST} ]; then
>>>>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>>>>> --add-architecture ${DISTRO_ARCH}
>>>>> + fi
>>>>> +
>>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>>> -o Debug::pkgProblemResolver=yes
>>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> index 73cd548..121992c 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>>> require buildchroot.inc
>>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>>> BUILDCHROOT_PREINSTALL ?= "make \
>>>>> debhelper \
>>>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>>>> automake \
>>>>> devscripts \
>>>>> equivs \
>>>>> - libc6:${DISTRO_ARCH}"
>>>>> -
>>>>> -# According to the wiki page:
>>>>> -# https://wiki.debian.org/CrossToolchains
>>>>> -BUILDCHROOT_PREINSTALL_append_armhf +=
>>>>> "binutils-arm-linux-gnueabihf \
>>>>> - crossbuild-essential-armhf"
>>>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>>>> - crossbuild-essential-arm64"
>>>>> -
>>>>> -
>>>>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-host:do_bootstrap"
>>>>> + libc6:${DISTRO_ARCH} \
>>>>> + crossbuild-essential-${DISTRO_ARCH}"
>>>>> diff --git
>>>>> a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> index 20f4d23..5a01258 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>>>>> devscripts \
>>>>> equivs \
>>>>> adduser"
>>>>> -
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-target:do_bootstrap"
>>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> index ca9ce51..40fc8b0 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>>>>> file://deps.sh"
>>>>> PV = "1.0"
>>>>> -inherit isar-bootstrap-helper
>>>>> +inherit rootfs
>>>>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>>>>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>>>>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>>>>> +ROOTFS_CLEAN_FILES = ""
>>>>> +
>>>>> +rootfs_do_mounts_append() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>>>>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>>>>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>>>>> +EOSUDO
>>>>> +}
>>>>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>>>>> - ${BUILDCHROOT_DIR}/isar-apt \
>>>>> - ${BUILDCHROOT_DIR}/downloads \
>>>>> - ${BUILDCHROOT_DIR}/home/builder"
>>>>> -
>>>>> -do_build() {
>>>>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>>>>> ${BUILDCHROOT_PREINSTALL}
>>>>> -
>>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>>>>> +buildchroot_install_files() {
>>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>>>>> # Install package builder script
>>>>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>>>>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>>>>> @@ -38,5 +45,4 @@ do_build() {
>>>>> GROUP_ID=$(id -g)
>>>>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID
>>>>> $GROUP_ID
>>>>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>>>>> }
>>>>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> index 49aeb47..57d52ba 100644
>>>>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> @@ -12,10 +12,25 @@ SRC_URI = " \
>>>>> file://configscript.sh \
>>>>> file://README.sdk"
>>>>> PV = "0.1"
>>>>> -
>>>>> -inherit isar-bootstrap-helper
>>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>>>>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>>>>> +TOOLCHAIN_i386 = "build-essential"
>>>>> +
>>>>> +inherit rootfs
>>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>>> +ROOTFSDIR = "${S}"
>>>>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>>>>> +ROOTFS_FEATURES += "copy-package-cache"
>>>>> +
>>>>> +python() {
>>>>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>>>>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>>>>> host".format(
>>>>> + d.getVar("ROOTFS_ARCH")))
>>>>> +}
>>>>> +
>>>>> SDKCHROOT_PREINSTALL := "debhelper \
>>>>> autotools-dev \
>>>>> dpkg \
>>>>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>>>>> S = "${WORKDIR}/rootfs"
>>>>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>>> -do_build[root_cleandirs] = "${S} \
>>>>> - ${S}/isar-apt"
>>>>> -
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-host:do_bootstrap"
>>>>> -
>>>>> -do_build() {
>>>>> -
>>>>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>>>>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>>>>> - fi
>>>>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>>>>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>>>>> - fi
>>>>> -
>>>>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386"
>>>>> ]; then
>>>>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>>>>> - else
>>>>> - packages="${SDKCHROOT_PREINSTALL}
>>>>> crossbuild-essential-${DISTRO_ARCH}"
>>>>> - fi
>>>>> -
>>>>> - setup_root_file_system --copyisarapt --host-arch
>>>>> '${HOST_ARCH}' --host-distro '${HOST_DISTRO}' "${S}" $packages
>>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>>>>> +rootfs_configure_isar_apt_dir() {
>>>>> + # Copy isar-apt instead of mounting:
>>>>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>>>>> +}
>>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>>>>> +sdkchroot_install_files() {
>>>>> # Configure root filesystem
>>>>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>>>>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>>>>
>>>>
>>>>
>>>
>>
>>
>
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-27 7:20 ` Maxim Yu. Osipov
2019-05-27 7:36 ` Maxim Yu. Osipov
@ 2019-05-27 9:03 ` Claudius Heine
2019-05-27 9:49 ` Maxim Yu. Osipov
1 sibling, 1 reply; 30+ messages in thread
From: Claudius Heine @ 2019-05-27 9:03 UTC (permalink / raw)
To: Maxim Yu. Osipov, isar-users; +Cc: Claudius Heine
Hi,
On 27/05/2019 09.20, Maxim Yu. Osipov wrote:
> Hi Claudius,
>
> On 5/27/19 8:55 AM, Claudius Heine wrote:
>> Hi Maxim,
>>
>> On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
>>> Hi Claudius,
>>>
>>> One may agree that setup_root_file_system is too "fat"
>>> and should be split to logical parts.
>>
>> There are multiple issues with `setup_root_file_system`, not just it
>> being 'fat'. For instance the amount of parameters this function
>> accepts just screams to me that it does not scale necessary
>> customization.
>>
>>> My concern is that one have to carefully keep track for every
>>> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of
>>> execution - possible source of errors.
>>
>> The weight parameter does not specify the order or execution, but just
>> the percentage of the progress bar shown when building. Since
>> do_rootfs_install is a pretty big task, I found it useful to show at
>> least some progress.
>
> Ah, I see now. Please add this comment into the code (to avoid confusion).
I think that could also be done later in an additional patchset, if that
is the only reason for a new series.
>
>>
>> The order of commands is the order in which they are listed in the
>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
>>
>> Currently that is not an interface for the user, but that might happen
>> if more variables are added there to signify different stages of the
>> rootfs build process, for instance how the
>> `ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
>
> I think that it's worth to add description of
> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND in RECIPE_API_CHANGELOG.
I can do that, but why? No user API has changed AFAIK and
ROOTFS_*_COMMAND aren't yet user API. They might become in the future,
but currently I would like to threat them as internal API since adding
to them from outside of isar is not very comfortable.
regards,
Claudius
>
> Regards,
> Maxim.
>
>
>> regards,
>> Claudius
>>
>>>
>>> Just a case: one my not define weight value (in such case default
>>> value is 20) as its is done for
>>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if
>>> we have several such default weights the order of commands could be
>>> wrong.
>>>
>>> Regards,
>>> Maxim.
>>>
>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>> From: Claudius Heine <ch@denx.de>
>>>>
>>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>>> functionality into the rootfs.bbclass.
>>>>
>>>> It introduces the rootfs_install task that runs through multiple
>>>> functions from command lists in order to allow easy integration of
>>>> custom functions.
>>>>
>>>> The introduced functions are, in general order of execution:
>>>>
>>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>>> rootfs_configure_*: basic configuration of the root file system,
>>>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>>>> are not mounted). Functions are added to
>>>> `ROOTFS_CONFIGURE_COMMAND`.
>>>> rootfs_do_mount: mount standard directories and repositories into
>>>> the
>>>> rootfs
>>>> do_rootfs_install_*: downloading and installing packages. Functions
>>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>>
>>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>>> ---
>>>> meta/classes/image.bbclass | 24 +--
>>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>>> meta/classes/rootfs.bbclass | 172
>>>> +++++++++++++++++-
>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>> .../buildchroot/buildchroot.inc | 26 ++-
>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>
>>>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>>>> index d77626c..99eea92 100644
>>>> --- a/meta/classes/image.bbclass
>>>> +++ b/meta/classes/image.bbclass
>>>> @@ -51,8 +51,10 @@ image_do_mounts() {
>>>> buildchroot_do_mounts
>>>> }
>>>> -inherit isar-bootstrap-helper
>>>> -ROOTFS_FEATURES += "finalize-rootfs"
>>>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>>>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>>>> finalize-rootfs"
>>>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>>> +
>>>> inherit rootfs
>>>> inherit image-sdk-extension
>>>> inherit image-cache-extension
>>>> @@ -119,8 +121,10 @@ python set_image_size () {
>>>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>>>> }
>>>> -do_image_gen_fstab() {
>>>> - cat > ${WORKDIR}/fstab << EOF
>>>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>>>> +image_configure_fstab[weight] = "2"
>>>> +image_configure_fstab() {
>>>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>>>> # Begin /etc/fstab
>>>> /dev/root / auto defaults 0 0
>>>> proc /proc proc nosuid,noexec,nodev 0 0
>>>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>>>> mode=0755,nosuid 0 0
>>>> # End /etc/fstab
>>>> EOF
>>>> }
>>>> -addtask image_gen_fstab before do_rootfs_install
>>>> -
>>>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-target:do_bootstrap"
>>>> -do_rootfs_install[deptask] = "do_deploy_deb"
>>>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>>>> - ${IMAGE_ROOTFS}/isar-apt"
>>>> -do_rootfs_install() {
>>>> - setup_root_file_system --clean --keep-apt-cache \
>>>> - --fstab "${WORKDIR}/fstab" \
>>>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>>>> -}
>>>> -addtask rootfs_install before do_build after do_unpack
>>>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>> do_copy_boot_files() {
>>>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>>>> b/meta/classes/isar-bootstrap-helper.bbclass
>>>> deleted file mode 100644
>>>> index bbc6322..0000000
>>>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>>>> +++ /dev/null
>>>> @@ -1,132 +0,0 @@
>>>> -# Helper functions for using isar-bootstrap
>>>> -#
>>>> -# This software is a part of ISAR.
>>>> -# Copyright (c) Siemens AG, 2018
>>>> -#
>>>> -# SPDX-License-Identifier: MIT
>>>> -
>>>> -def reverse_bb_array(d, varname):
>>>> - array = d.getVar(varname, True)
>>>> - if array is None:
>>>> - return None
>>>> - array = reversed(array.split())
>>>> - return " ".join(i for i in array)
>>>> -
>>>> -
>>>> -update_etc_os_release() {
>>>> - OS_RELEASE_BUILD_ID=""
>>>> - OS_RELEASE_VARIANT=""
>>>> - while true; do
>>>> - case "$1" in
>>>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>>>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>> - *) break ;;
>>>> - esac
>>>> - shift
>>>> - done
>>>> - ROOTFSDIR="$1"
>>>> -
>>>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>>>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>>>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>> - fi
>>>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>>>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>>>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>> - fi
>>>> -}
>>>> -
>>>> -setup_root_file_system() {
>>>> - CLEAN=""
>>>> - COPYISARAPT=""
>>>> - FSTAB=""
>>>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>>>> - ROOTFS_DISTRO="${DISTRO}"
>>>> - while true; do
>>>> - case "$1" in
>>>> - --clean) CLEAN=1 ;;
>>>> - --copyisarapt) COPYISARAPT=1 ;;
>>>> - --fstab) FSTAB=$2; shift ;;
>>>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>>>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>>>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>> - *) break ;;
>>>> - esac
>>>> - shift
>>>> - done
>>>> - ROOTFSDIR="$1"
>>>> - shift
>>>> - PACKAGES="$@"
>>>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname
>>>> ${ROOTFSDIR}/etc/resolv.conf"
>>>> -
>>>> - sudo cp -Trpfx \
>>>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>>>> - "$ROOTFSDIR"
>>>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee
>>>> "$ROOTFSDIR/etc/fstab"
>>>> -
>>>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main"
>>>> | \
>>>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>>>> >/dev/null
>>>> -
>>>> - echo "Package: *\nPin: release
>>>> n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
>>>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>>>> -
>>>> - if [ ${COPYISARAPT} ]; then
>>>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>>>> - else
>>>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO}
>>>> $ROOTFSDIR/isar-apt
>>>> - fi
>>>> -
>>>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>>>> - fi
>>>> -
>>>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>>>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>>>> - sudo mount -t proc none $ROOTFSDIR/proc
>>>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>>>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>>>> -
>>>> - # Refresh /etc/resolv.conf
>>>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>>>> -
>>>> - # Install packages:
>>>> - E="${@ bb.utils.export_proxies(d)}"
>>>> - export DEBIAN_FRONTEND=noninteractive
>>>> - # To avoid Perl locale warnings:
>>>> - export LANG=C
>>>> - export LANGUAGE=C
>>>> - export LC_ALL=C
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>>>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>> - -o Dir::Etc::sourceparts="-" \
>>>> - -o APT::Get::List-Cleanup="0"
>>>> - # Add multiarch for cross-target
>>>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg
>>>> --add-architecture ${DISTRO_ARCH}
>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>>>> - fi
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>>>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>>>> - if [ ${CLEAN} ]; then
>>>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>>>> - mkdir -p ${WORKDIR}/apt_cache
>>>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>>>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>> ${WORKDIR}/apt_cache '{}' '+'
>>>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>>>> - fi
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get autoremove --purge --yes
>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>> - /usr/bin/apt-get clean
>>>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>>>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>>>> "$ROOTFSDIR/var/lib/apt/lists/"*
>>>> - fi
>>>> -}
>>>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>>>> index 45b0350..0fc8ba8 100644
>>>> --- a/meta/classes/rootfs.bbclass
>>>> +++ b/meta/classes/rootfs.bbclass
>>>> @@ -1,14 +1,178 @@
>>>> # This software is a part of ISAR.
>>>> # Copyright (c) Siemens AG, 2019
>>>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>>>> +ROOTFS_DISTRO ?= "${DISTRO}"
>>>> +ROOTFS_PACKAGES ?= ""
>>>> +
>>>> # Features of the rootfs creation:
>>>> # available features are:
>>>> +# 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
>>>> +# 'clean-package-cache' - delete package cache from rootfs
>>>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>>>> ROOTFS_FEATURES ?= ""
>>>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>> +
>>>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>>>> +
>>>> +# Useful environment variables:
>>>> +export E = "${@ bb.utils.export_proxies(d)}"
>>>> +export DEBIAN_FRONTEND = "noninteractive"
>>>> +# To avoid Perl locale warnings:
>>>> +export LANG = "C"
>>>> +export LANGUAGE = "C"
>>>> +export LC_ALL = "C"
>>>> +
>>>> +rootfs_do_mounts[weight] = "3"
>>>> +rootfs_do_mounts() {
>>>> + sudo -s <<'EOSUDO'
>>>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>>>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>>>> + mount --make-rslave '${ROOTFSDIR}/dev'
>>>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>>>> + mount -t proc none '${ROOTFSDIR}/proc'
>>>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>>>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>>>> + mount --make-rslave '${ROOTFSDIR}/sys'
>>>> +
>>>> + # Mount isar-apt if the directory does not exist or if it
>>>> is empty
>>>> + # This prevents overwriting something that was copied there
>>>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>>>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth
>>>> 1 | wc -l)" = "0" ]
>>>> + then
>>>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>>>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>>>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>>>> '${ROOTFSDIR}/isar-apt'
>>>> + fi
>>>> +
>>>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>>>> + if [
>>>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}"
>>>> = 'True' ]
>>>> + then
>>>> + mkdir -p '${ROOTFSDIR}/base-apt'
>>>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>>>> + mount --bind '${REPO_BASE_DIR}'
>>>> '${ROOTFSDIR}/base-apt'
>>>> + fi
>>>> +
>>>> +EOSUDO
>>>> +}
>>>> +
>>>> +rootfs_do_qemu() {
>>>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') ==
>>>> d.getVar('HOST_ARCH'))}' = 'False' ]
>>>> + then
>>>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>>>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>>>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>>>> + fi
>>>> +}
>>>> +
>>>> +BOOTSTRAP_SRC =
>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>>>>
>>>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>>>> +
>>>> +rootfs_prepare[weight] = "25"
>>>> +rootfs_prepare(){
>>>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>>>> +}
>>>> +
>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>>>> +rootfs_configure_isar_apt[weight] = "2"
>>>> +rootfs_configure_isar_apt() {
>>>> + sudo -s <<'EOSUDO'
>>>> +
>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>>>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main'
>>>> > \
>>>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>>>> +
>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>>>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>>>> +Package: *
>>>> +Pin: release n=${DEBDISTRONAME}
>>>> +Pin-Priority: 1000
>>>> +EOF
>>>> +EOSUDO
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>>>> +rootfs_install_pkgs_update[weight] = "5"
>>>> +rootfs_install_pkgs_update() {
>>>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>>>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>> + -o Dir::Etc::sourceparts="-" \
>>>> + -o APT::Get::List-Cleanup="0"
>>>> +}
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>>>> +rootfs_install_resolvconf[weight] = "1"
>>>> +rootfs_install_resolvconf() {
>>>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>>>> +rootfs_install_pkgs_download[weight] = "600"
>>>> +rootfs_install_pkgs_download() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>>>> ${ROOTFS_PACKAGES}
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>>>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>>>> +rootfs_install_clean_files[weight] = "2"
>>>> +rootfs_install_clean_files() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>>>> +}
>>>> +
>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>>>> +rootfs_install_pkgs_install[weight] = "8000"
>>>> +rootfs_install_pkgs_install() {
>>>> + sudo -E chroot "${ROOTFSDIR}" \
>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>>>> +}
>>>
>>>
>>> Honestly I don't see benefits in such multiple
>>> configure_cmds/rootfs_install_cmds - it doesn't increase code
>>> readability as one have to additionally keep track for every command
>>> 'weight' value and it's easy to miss the correct order of commands
>>> execution.
>>>
>>>
>>>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>>>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>>>> ${ROOTFS_INSTALL_COMMAND}"
>>>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>>>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>>>> 'host'}:do_build isar-apt:do_cache_config"
>>>> +do_rootfs_install[deptask] = "do_deploy_deb"
>>>> +python do_rootfs_install() {
>>>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or
>>>> "").split()
>>>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>>>> "").split()
>>>> +
>>>> + # Mount after configure commands, so that they have time to copy
>>>> + # 'isar-apt' (sdkchroot):
>>>> + cmds = ['rootfs_prepare'] + configure_cmds +
>>>> ['rootfs_do_mounts'] + install_cmds
>>>> +
>>>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>>>> + for i in cmds]
>>>> +
>>>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>>>> stage_weights)
>>>> +
>>>> + for cmd in cmds:
>>>> + progress_reporter.next_stage()
>>>> + bb.build.exec_func(cmd, d)
>>>> + progress_reporter.finish()
>>>> +}
>>>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>>>> +
>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>>>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>>>> +rootfs_postprocess_copy_package_cache() {
>>>> + mkdir -p '${WORKDIR}/apt_cache'
>>>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>>>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>> '${WORKDIR}/apt_cache' '{}' '+'
>>>> + me="$(id -u):$(id -g)"
>>>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>>>> +}
>>>> +
>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>>>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>>>> +rootfs_postprocess_clean_package_cache() {
>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>> + /usr/bin/apt-get clean
>>>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>>>> +}
>>>> +
>>>> ROOTFS_POSTPROCESS_COMMAND +=
>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>>>> 'rootfs_postprocess_finalize', '', d)}"
>>>> rootfs_postprocess_finalize() {
>>>> sudo -s <<'EOSUDO'
>>>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>>>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>>>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>>>> +
>>>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>>>> find "${ROOTFSDIR}/usr/bin" \
>>>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>>>> @@ -42,6 +206,12 @@ EOSUDO
>>>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>>>> python do_rootfs_postprocess() {
>>>> + # Take care that its correctly mounted:
>>>> + bb.build.exec_func('rootfs_do_mounts', d)
>>>> + # Take care that qemu-*-static is available, since it could
>>>> have been
>>>> + # removed on a previous execution of this task:
>>>> + bb.build.exec_func('rootfs_do_qemu', d)
>>>> +
>>>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>>>> if cmds is None or not cmds.strip():
>>>> return
>>>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>>>> for cmd in cmds:
>>>> bb.build.exec_func(cmd, d)
>>>> }
>>>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>>>> +addtask rootfs_postprocess before do_rootfs
>>>> python do_rootfs() {
>>>> """Virtual task"""
>>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> index b740c57..f8741c2 100644
>>>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>>>> mount --make-rslave ${ROOTFSDIR}/sys
>>>> export DEBIAN_FRONTEND=noninteractive
>>>> +
>>>> + if [ ${IS_HOST} ]; then
>>>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>>>> --add-architecture ${DISTRO_ARCH}
>>>> + fi
>>>> +
>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>> -o Debug::pkgProblemResolver=yes
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> index 73cd548..121992c 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>> require buildchroot.inc
>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>> BUILDCHROOT_PREINSTALL ?= "make \
>>>> debhelper \
>>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>>> automake \
>>>> devscripts \
>>>> equivs \
>>>> - libc6:${DISTRO_ARCH}"
>>>> -
>>>> -# According to the wiki page:
>>>> -# https://wiki.debian.org/CrossToolchains
>>>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>>>> - crossbuild-essential-armhf"
>>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>>> - crossbuild-essential-arm64"
>>>> -
>>>> -
>>>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-host:do_bootstrap"
>>>> + libc6:${DISTRO_ARCH} \
>>>> + crossbuild-essential-${DISTRO_ARCH}"
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> index 20f4d23..5a01258 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>>>> devscripts \
>>>> equivs \
>>>> adduser"
>>>> -
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-target:do_bootstrap"
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> index ca9ce51..40fc8b0 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>>>> file://deps.sh"
>>>> PV = "1.0"
>>>> -inherit isar-bootstrap-helper
>>>> +inherit rootfs
>>>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>>>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>>>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>>>> +ROOTFS_CLEAN_FILES = ""
>>>> +
>>>> +rootfs_do_mounts_append() {
>>>> + sudo -s <<'EOSUDO'
>>>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>>>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>>>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>>>> +EOSUDO
>>>> +}
>>>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>>>> - ${BUILDCHROOT_DIR}/isar-apt \
>>>> - ${BUILDCHROOT_DIR}/downloads \
>>>> - ${BUILDCHROOT_DIR}/home/builder"
>>>> -
>>>> -do_build() {
>>>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>>>> ${BUILDCHROOT_PREINSTALL}
>>>> -
>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>>>> +buildchroot_install_files() {
>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>>>> # Install package builder script
>>>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>>>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>>>> @@ -38,5 +45,4 @@ do_build() {
>>>> GROUP_ID=$(id -g)
>>>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID
>>>> $GROUP_ID
>>>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>>>> }
>>>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> index 49aeb47..57d52ba 100644
>>>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>> @@ -12,10 +12,25 @@ SRC_URI = " \
>>>> file://configscript.sh \
>>>> file://README.sdk"
>>>> PV = "0.1"
>>>> -
>>>> -inherit isar-bootstrap-helper
>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>>>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>>>> +TOOLCHAIN_i386 = "build-essential"
>>>> +
>>>> +inherit rootfs
>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>> +ROOTFSDIR = "${S}"
>>>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>>>> +ROOTFS_FEATURES += "copy-package-cache"
>>>> +
>>>> +python() {
>>>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>>>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>>>> host".format(
>>>> + d.getVar("ROOTFS_ARCH")))
>>>> +}
>>>> +
>>>> SDKCHROOT_PREINSTALL := "debhelper \
>>>> autotools-dev \
>>>> dpkg \
>>>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>>>> S = "${WORKDIR}/rootfs"
>>>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>> -do_build[root_cleandirs] = "${S} \
>>>> - ${S}/isar-apt"
>>>> -
>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>> isar-bootstrap-host:do_bootstrap"
>>>> -
>>>> -do_build() {
>>>> -
>>>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>>>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>>>> - fi
>>>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>>>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>>>> - fi
>>>> -
>>>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386"
>>>> ]; then
>>>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>>>> - else
>>>> - packages="${SDKCHROOT_PREINSTALL}
>>>> crossbuild-essential-${DISTRO_ARCH}"
>>>> - fi
>>>> -
>>>> - setup_root_file_system --copyisarapt --host-arch '${HOST_ARCH}'
>>>> --host-distro '${HOST_DISTRO}' "${S}" $packages
>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>>>> +rootfs_configure_isar_apt_dir() {
>>>> + # Copy isar-apt instead of mounting:
>>>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>>>> +}
>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>>>> +sdkchroot_install_files() {
>>>> # Configure root filesystem
>>>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>>>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>>>
>>>
>>>
>>
>
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-27 9:03 ` Claudius Heine
@ 2019-05-27 9:49 ` Maxim Yu. Osipov
2019-05-27 10:44 ` [PATCH] rootfs.bbclass: add comment about task weights claudius.heine.ext
0 siblings, 1 reply; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 9:49 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
On 5/27/19 11:03 AM, Claudius Heine wrote:
> Hi,
>
> On 27/05/2019 09.20, Maxim Yu. Osipov wrote:
>> Hi Claudius,
>>
>> On 5/27/19 8:55 AM, Claudius Heine wrote:
>>> Hi Maxim,
>>>
>>> On 24/05/2019 14.49, Maxim Yu. Osipov wrote:
>>>> Hi Claudius,
>>>>
>>>> One may agree that setup_root_file_system is too "fat"
>>>> and should be split to logical parts.
>>>
>>> There are multiple issues with `setup_root_file_system`, not just it
>>> being 'fat'. For instance the amount of parameters this function
>>> accepts just screams to me that it does not scale necessary
>>> customization.
>>>
>>>> My concern is that one have to carefully keep track for every
>>>> ROOTFS_CONFIGURE/ROOTFS_INSTALL command's 'weight' - order of
>>>> execution - possible source of errors.
>>>
>>> The weight parameter does not specify the order or execution, but
>>> just the percentage of the progress bar shown when building. Since
>>> do_rootfs_install is a pretty big task, I found it useful to show at
>>> least some progress.
>>
>> Ah, I see now. Please add this comment into the code (to avoid
>> confusion).
>
> I think that could also be done later in an additional patchset, if that
> is the only reason for a new series.
Sure - new series are not required. - just add the more comments in the
appropriate place in the code.
>>
>>>
>>> The order of commands is the order in which they are listed in the
>>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND array.
>>>
>>> Currently that is not an interface for the user, but that might
>>> happen if more variables are added there to signify different stages
>>> of the rootfs build process, for instance how the
>>> `ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN` was done.
>>
>> I think that it's worth to add description of
>> ROOTFS_CONFIGURE_COMMAND/ROOTFS_INSTALL_COMMAND in RECIPE_API_CHANGELOG.
>
> I can do that, but why? No user API has changed AFAIK and
> ROOTFS_*_COMMAND aren't yet user API. They might become in the future,
> but currently I would like to threat them as internal API since adding
> to them from outside of isar is not very comfortable.
I hope treat and not threat ;)
Regards,
Maxim.
> regards,
> Claudius
>
>>
>> Regards,
>> Maxim.
>>
>>
>>> regards,
>>> Claudius
>>>
>>>>
>>>> Just a case: one my not define weight value (in such case default
>>>> value is 20) as its is done for
>>>> ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir" - so if
>>>> we have several such default weights the order of commands could be
>>>> wrong.
>>>>
>>>> Regards,
>>>> Maxim.
>>>>
>>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>>> From: Claudius Heine <ch@denx.de>
>>>>>
>>>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>>>> functionality into the rootfs.bbclass.
>>>>>
>>>>> It introduces the rootfs_install task that runs through multiple
>>>>> functions from command lists in order to allow easy integration of
>>>>> custom functions.
>>>>>
>>>>> The introduced functions are, in general order of execution:
>>>>>
>>>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>>>> rootfs_configure_*: basic configuration of the root file system,
>>>>> before any custom packages where installed. (`/dev`, `/proc`,
>>>>> etc.
>>>>> are not mounted). Functions are added to
>>>>> `ROOTFS_CONFIGURE_COMMAND`.
>>>>> rootfs_do_mount: mount standard directories and repositories
>>>>> into the
>>>>> rootfs
>>>>> do_rootfs_install_*: downloading and installing packages. Functions
>>>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>>>
>>>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>>>> ---
>>>>> meta/classes/image.bbclass | 24 +--
>>>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>>>> meta/classes/rootfs.bbclass | 172
>>>>> +++++++++++++++++-
>>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>>> .../buildchroot/buildchroot.inc | 26 ++-
>>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>>
>>>>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>>>>> index d77626c..99eea92 100644
>>>>> --- a/meta/classes/image.bbclass
>>>>> +++ b/meta/classes/image.bbclass
>>>>> @@ -51,8 +51,10 @@ image_do_mounts() {
>>>>> buildchroot_do_mounts
>>>>> }
>>>>> -inherit isar-bootstrap-helper
>>>>> -ROOTFS_FEATURES += "finalize-rootfs"
>>>>> +ROOTFSDIR = "${IMAGE_ROOTFS}"
>>>>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache
>>>>> finalize-rootfs"
>>>>> +ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>>>> +
>>>>> inherit rootfs
>>>>> inherit image-sdk-extension
>>>>> inherit image-cache-extension
>>>>> @@ -119,8 +121,10 @@ python set_image_size () {
>>>>> d.setVarFlag('ROOTFS_SIZE', 'export', '1')
>>>>> }
>>>>> -do_image_gen_fstab() {
>>>>> - cat > ${WORKDIR}/fstab << EOF
>>>>> +ROOTFS_CONFIGURE_COMMAND += "image_configure_fstab"
>>>>> +image_configure_fstab[weight] = "2"
>>>>> +image_configure_fstab() {
>>>>> + sudo tee '${IMAGE_ROOTFS}/etc/fstab' << EOF
>>>>> # Begin /etc/fstab
>>>>> /dev/root / auto defaults 0 0
>>>>> proc /proc proc nosuid,noexec,nodev 0 0
>>>>> @@ -132,18 +136,6 @@ devtmpfs /dev devtmpfs
>>>>> mode=0755,nosuid 0 0
>>>>> # End /etc/fstab
>>>>> EOF
>>>>> }
>>>>> -addtask image_gen_fstab before do_rootfs_install
>>>>> -
>>>>> -do_rootfs_install[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-target:do_bootstrap"
>>>>> -do_rootfs_install[deptask] = "do_deploy_deb"
>>>>> -do_rootfs_install[root_cleandirs] = "${IMAGE_ROOTFS} \
>>>>> - ${IMAGE_ROOTFS}/isar-apt"
>>>>> -do_rootfs_install() {
>>>>> - setup_root_file_system --clean --keep-apt-cache \
>>>>> - --fstab "${WORKDIR}/fstab" \
>>>>> - "${IMAGE_ROOTFS}" ${IMAGE_PREINSTALL} ${IMAGE_INSTALL}
>>>>> -}
>>>>> -addtask rootfs_install before do_build after do_unpack
>>>>> do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>>> do_copy_boot_files() {
>>>>> diff --git a/meta/classes/isar-bootstrap-helper.bbclass
>>>>> b/meta/classes/isar-bootstrap-helper.bbclass
>>>>> deleted file mode 100644
>>>>> index bbc6322..0000000
>>>>> --- a/meta/classes/isar-bootstrap-helper.bbclass
>>>>> +++ /dev/null
>>>>> @@ -1,132 +0,0 @@
>>>>> -# Helper functions for using isar-bootstrap
>>>>> -#
>>>>> -# This software is a part of ISAR.
>>>>> -# Copyright (c) Siemens AG, 2018
>>>>> -#
>>>>> -# SPDX-License-Identifier: MIT
>>>>> -
>>>>> -def reverse_bb_array(d, varname):
>>>>> - array = d.getVar(varname, True)
>>>>> - if array is None:
>>>>> - return None
>>>>> - array = reversed(array.split())
>>>>> - return " ".join(i for i in array)
>>>>> -
>>>>> -
>>>>> -update_etc_os_release() {
>>>>> - OS_RELEASE_BUILD_ID=""
>>>>> - OS_RELEASE_VARIANT=""
>>>>> - while true; do
>>>>> - case "$1" in
>>>>> - --build-id) OS_RELEASE_BUILD_ID=$2; shift ;;
>>>>> - --variant) OS_RELEASE_VARIANT=$2; shift ;;
>>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>>> - *) break ;;
>>>>> - esac
>>>>> - shift
>>>>> - done
>>>>> - ROOTFSDIR="$1"
>>>>> -
>>>>> - if [ -n "${OS_RELEASE_BUILD_ID}" ]; then
>>>>> - sudo sed -i '/^BUILD_ID=.*/d' ${ROOTFSDIR}/etc/os-release
>>>>> - echo "BUILD_ID=\"${OS_RELEASE_BUILD_ID}\"" | \
>>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>>> - fi
>>>>> - if [ -n "${OS_RELEASE_VARIANT}" ]; then
>>>>> - sudo sed -i '/^VARIANT=.*/d' ${ROOTFSDIR}/etc/os-release
>>>>> - echo "VARIANT=\"${OS_RELEASE_VARIANT}\"" | \
>>>>> - sudo tee -a ${ROOTFSDIR}/etc/os-release
>>>>> - fi
>>>>> -}
>>>>> -
>>>>> -setup_root_file_system() {
>>>>> - CLEAN=""
>>>>> - COPYISARAPT=""
>>>>> - FSTAB=""
>>>>> - ROOTFS_ARCH="${DISTRO_ARCH}"
>>>>> - ROOTFS_DISTRO="${DISTRO}"
>>>>> - while true; do
>>>>> - case "$1" in
>>>>> - --clean) CLEAN=1 ;;
>>>>> - --copyisarapt) COPYISARAPT=1 ;;
>>>>> - --fstab) FSTAB=$2; shift ;;
>>>>> - --host-arch) ROOTFS_ARCH="$2-$ROOTFS_ARCH"; shift ;;
>>>>> - --host-distro) ROOTFS_DISTRO="$2"; shift ;;
>>>>> - --keep-apt-cache) KEEP_APT_CACHE=1 ;;
>>>>> - -*) bbfatal "$0: invalid option specified: $1" ;;
>>>>> - *) break ;;
>>>>> - esac
>>>>> - shift
>>>>> - done
>>>>> - ROOTFSDIR="$1"
>>>>> - shift
>>>>> - PACKAGES="$@"
>>>>> - APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>>> - CLEAN_FILES="${ROOTFSDIR}/etc/hostname
>>>>> ${ROOTFSDIR}/etc/resolv.conf"
>>>>> -
>>>>> - sudo cp -Trpfx \
>>>>> - "${DEPLOY_DIR_BOOTSTRAP}/$ROOTFS_DISTRO-$ROOTFS_ARCH/" \
>>>>> - "$ROOTFSDIR"
>>>>> - [ -n "${FSTAB}" ] && cat ${FSTAB} | sudo tee
>>>>> "$ROOTFSDIR/etc/fstab"
>>>>> -
>>>>> - echo "deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME}
>>>>> main" | \
>>>>> - sudo tee "$ROOTFSDIR/etc/apt/sources.list.d/isar-apt.list"
>>>>> >/dev/null
>>>>> -
>>>>> - echo "Package: *\nPin: release
>>>>> n=${DEBDISTRONAME}\nPin-Priority: 1000" | \
>>>>> - sudo tee "$ROOTFSDIR/etc/apt/preferences.d/isar" >/dev/null
>>>>> -
>>>>> - if [ ${COPYISARAPT} ]; then
>>>>> - sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} $ROOTFSDIR/isar-apt
>>>>> - else
>>>>> - sudo mount --bind ${REPO_ISAR_DIR}/${DISTRO}
>>>>> $ROOTFSDIR/isar-apt
>>>>> - fi
>>>>> -
>>>>> - if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
>>>>> - sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
>>>>> - fi
>>>>> -
>>>>> - sudo mount --rbind /dev ${ROOTFSDIR}/dev
>>>>> - sudo mount --make-rslave ${ROOTFSDIR}/dev
>>>>> - sudo mount -t proc none $ROOTFSDIR/proc
>>>>> - sudo mount --rbind /sys ${ROOTFSDIR}/sys
>>>>> - sudo mount --make-rslave ${ROOTFSDIR}/sys
>>>>> -
>>>>> - # Refresh /etc/resolv.conf
>>>>> - sudo cp -L /etc/resolv.conf ${ROOTFSDIR}/etc
>>>>> -
>>>>> - # Install packages:
>>>>> - E="${@ bb.utils.export_proxies(d)}"
>>>>> - export DEBIAN_FRONTEND=noninteractive
>>>>> - # To avoid Perl locale warnings:
>>>>> - export LANG=C
>>>>> - export LANGUAGE=C
>>>>> - export LC_ALL=C
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update \
>>>>> - -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>>> - -o Dir::Etc::sourceparts="-" \
>>>>> - -o APT::Get::List-Cleanup="0"
>>>>> - # Add multiarch for cross-target
>>>>> - if [ "${ROOTFS_ARCH}" != "${DISTRO_ARCH}" ]; then
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/dpkg
>>>>> --add-architecture ${DISTRO_ARCH}
>>>>> - sudo -E chroot "$ROOTFSDIR" /usr/bin/apt-get update
>>>>> - fi
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get ${APT_ARGS} --download-only $PACKAGES
>>>>> - [ ${CLEAN} ] && sudo rm -f ${CLEAN_FILES}
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get ${APT_ARGS} $PACKAGES
>>>>> - if [ ${CLEAN} ]; then
>>>>> - if [ ${KEEP_APT_CACHE} -eq 1 ]; then
>>>>> - mkdir -p ${WORKDIR}/apt_cache
>>>>> - sudo find ${ROOTFSDIR}/var/cache/apt/archives \
>>>>> - -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>>> ${WORKDIR}/apt_cache '{}' '+'
>>>>> - sudo chown -R $(whoami) ${WORKDIR}/apt_cache
>>>>> - fi
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get autoremove --purge --yes
>>>>> - sudo -E chroot "$ROOTFSDIR" \
>>>>> - /usr/bin/apt-get clean
>>>>> - sudo "$ROOTFSDIR/chroot-setup.sh" "cleanup" "$ROOTFSDIR"
>>>>> - sudo rm -rf "$ROOTFSDIR/chroot-setup.sh"
>>>>> "$ROOTFSDIR/var/lib/apt/lists/"*
>>>>> - fi
>>>>> -}
>>>>> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
>>>>> index 45b0350..0fc8ba8 100644
>>>>> --- a/meta/classes/rootfs.bbclass
>>>>> +++ b/meta/classes/rootfs.bbclass
>>>>> @@ -1,14 +1,178 @@
>>>>> # This software is a part of ISAR.
>>>>> # Copyright (c) Siemens AG, 2019
>>>>> +ROOTFS_ARCH ?= "${DISTRO_ARCH}"
>>>>> +ROOTFS_DISTRO ?= "${DISTRO}"
>>>>> +ROOTFS_PACKAGES ?= ""
>>>>> +
>>>>> # Features of the rootfs creation:
>>>>> # available features are:
>>>>> +# 'deploy-package-cache' - copy the package cache
>>>>> ${WORKDIR}/apt_cache
>>>>> +# 'clean-package-cache' - delete package cache from rootfs
>>>>> # 'finalize-rootfs' - delete files needed to chroot into the rootfs
>>>>> ROOTFS_FEATURES ?= ""
>>>>> +ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>>>>> +
>>>>> +ROOTFS_CLEAN_FILES="/etc/hostname /etc/resolv.conf"
>>>>> +
>>>>> +# Useful environment variables:
>>>>> +export E = "${@ bb.utils.export_proxies(d)}"
>>>>> +export DEBIAN_FRONTEND = "noninteractive"
>>>>> +# To avoid Perl locale warnings:
>>>>> +export LANG = "C"
>>>>> +export LANGUAGE = "C"
>>>>> +export LC_ALL = "C"
>>>>> +
>>>>> +rootfs_do_mounts[weight] = "3"
>>>>> +rootfs_do_mounts() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> + mountpoint -q '${ROOTFSDIR}/dev' || \
>>>>> + mount --rbind /dev '${ROOTFSDIR}/dev'
>>>>> + mount --make-rslave '${ROOTFSDIR}/dev'
>>>>> + mountpoint -q '${ROOTFSDIR}/proc' || \
>>>>> + mount -t proc none '${ROOTFSDIR}/proc'
>>>>> + mountpoint -q '${ROOTFSDIR}/sys' || \
>>>>> + mount --rbind /sys '${ROOTFSDIR}/sys'
>>>>> + mount --make-rslave '${ROOTFSDIR}/sys'
>>>>> +
>>>>> + # Mount isar-apt if the directory does not exist or if it
>>>>> is empty
>>>>> + # This prevents overwriting something that was copied there
>>>>> + if [ ! -e '${ROOTFSDIR}/isar-apt' ] || \
>>>>> + [ "$(find '${ROOTFSDIR}/isar-apt' -maxdepth 1 -mindepth
>>>>> 1 | wc -l)" = "0" ]
>>>>> + then
>>>>> + mkdir -p '${ROOTFSDIR}/isar-apt'
>>>>> + mountpoint -q '${ROOTFSDIR}/isar-apt' || \
>>>>> + mount --bind '${REPO_ISAR_DIR}/${DISTRO}'
>>>>> '${ROOTFSDIR}/isar-apt'
>>>>> + fi
>>>>> +
>>>>> + # Mount base-apt if 'ISAR_USE_CACHED_BASE_REPO' is set
>>>>> + if [
>>>>> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')))}"
>>>>> = 'True' ]
>>>>> + then
>>>>> + mkdir -p '${ROOTFSDIR}/base-apt'
>>>>> + mountpoint -q '${ROOTFSDIR}/base-apt' || \
>>>>> + mount --bind '${REPO_BASE_DIR}'
>>>>> '${ROOTFSDIR}/base-apt'
>>>>> + fi
>>>>> +
>>>>> +EOSUDO
>>>>> +}
>>>>> +
>>>>> +rootfs_do_qemu() {
>>>>> + if [ '${@repr(d.getVar('ROOTFS_ARCH') ==
>>>>> d.getVar('HOST_ARCH'))}' = 'False' ]
>>>>> + then
>>>>> + test -e '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static' || \
>>>>> + sudo cp '/usr/bin/qemu-${QEMU_ARCH}-static'
>>>>> '${ROOTFSDIR}/usr/bin/qemu-${QEMU_ARCH}-static'
>>>>> + fi
>>>>> +}
>>>>> +
>>>>> +BOOTSTRAP_SRC =
>>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}-${DISTRO_ARCH}"
>>>>>
>>>>> +BOOTSTRAP_SRC_${ROOTFS_ARCH} =
>>>>> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>>>>> +
>>>>> +rootfs_prepare[weight] = "25"
>>>>> +rootfs_prepare(){
>>>>> + sudo cp -Trpfx '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt"
>>>>> +rootfs_configure_isar_apt[weight] = "2"
>>>>> +rootfs_configure_isar_apt() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> +
>>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
>>>>> + echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME}
>>>>> main' > \
>>>>> + '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
>>>>> +
>>>>> + mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
>>>>> + cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar'
>>>>> +Package: *
>>>>> +Pin: release n=${DEBDISTRONAME}
>>>>> +Pin-Priority: 1000
>>>>> +EOF
>>>>> +EOSUDO
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_update"
>>>>> +rootfs_install_pkgs_update[weight] = "5"
>>>>> +rootfs_install_pkgs_update() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' /usr/bin/apt-get update \
>>>>> + -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
>>>>> + -o Dir::Etc::sourceparts="-" \
>>>>> + -o APT::Get::List-Cleanup="0"
>>>>> +}
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
>>>>> +rootfs_install_resolvconf[weight] = "1"
>>>>> +rootfs_install_resolvconf() {
>>>>> + sudo cp -rL /etc/resolv.conf '${ROOTFSDIR}/etc'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>>>>> +rootfs_install_pkgs_download[weight] = "600"
>>>>> +rootfs_install_pkgs_download() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
>>>>> ${ROOTFS_PACKAGES}
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
>>>>> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
>>>>> +rootfs_install_clean_files[weight] = "2"
>>>>> +rootfs_install_clean_files() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /bin/rm -f ${ROOTFS_CLEAN_FILES}
>>>>> +}
>>>>> +
>>>>> +ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>>>>> +rootfs_install_pkgs_install[weight] = "8000"
>>>>> +rootfs_install_pkgs_install() {
>>>>> + sudo -E chroot "${ROOTFSDIR}" \
>>>>> + /usr/bin/apt-get ${ROOTFS_APT_ARGS} ${ROOTFS_PACKAGES}
>>>>> +}
>>>>
>>>>
>>>> Honestly I don't see benefits in such multiple
>>>> configure_cmds/rootfs_install_cmds - it doesn't increase code
>>>> readability as one have to additionally keep track for every command
>>>> 'weight' value and it's easy to miss the correct order of commands
>>>> execution.
>>>>
>>>>
>>>>> +do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
>>>>> +do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
>>>>> ${ROOTFS_INSTALL_COMMAND}"
>>>>> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
>>>>> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
>>>>> 'host'}:do_build isar-apt:do_cache_config"
>>>>> +do_rootfs_install[deptask] = "do_deploy_deb"
>>>>> +python do_rootfs_install() {
>>>>> + configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True)
>>>>> or "").split()
>>>>> + install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND", True) or
>>>>> "").split()
>>>>> +
>>>>> + # Mount after configure commands, so that they have time to copy
>>>>> + # 'isar-apt' (sdkchroot):
>>>>> + cmds = ['rootfs_prepare'] + configure_cmds +
>>>>> ['rootfs_do_mounts'] + install_cmds
>>>>> +
>>>>> + stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
>>>>> + for i in cmds]
>>>>> +
>>>>> + progress_reporter = bb.progress.MultiStageProgressReporter(d,
>>>>> stage_weights)
>>>>> +
>>>>> + for cmd in cmds:
>>>>> + progress_reporter.next_stage()
>>>>> + bb.build.exec_func(cmd, d)
>>>>> + progress_reporter.finish()
>>>>> +}
>>>>> +addtask rootfs_install before do_rootfs_postprocess after do_unpack
>>>>> +
>>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache',
>>>>> 'rootfs_postprocess_copy_package_cache', '', d)}"
>>>>> +rootfs_postprocess_copy_package_cache() {
>>>>> + mkdir -p '${WORKDIR}/apt_cache'
>>>>> + sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
>>>>> + -maxdepth 1 -name '*.deb' -execdir /bin/mv -t
>>>>> '${WORKDIR}/apt_cache' '{}' '+'
>>>>> + me="$(id -u):$(id -g)"
>>>>> + sudo chown -R "$me" '${WORKDIR}/apt_cache'
>>>>> +}
>>>>> +
>>>>> +ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache',
>>>>> 'rootfs_postprocess_clean_package_cache', '', d)}"
>>>>> +rootfs_postprocess_clean_package_cache() {
>>>>> + sudo -E chroot '${ROOTFSDIR}' \
>>>>> + /usr/bin/apt-get clean
>>>>> + sudo rm -rf "${ROOTFSDIR}/var/lib/apt/lists/"*
>>>>> +}
>>>>> +
>>>>> ROOTFS_POSTPROCESS_COMMAND +=
>>>>> "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs',
>>>>> 'rootfs_postprocess_finalize', '', d)}"
>>>>> rootfs_postprocess_finalize() {
>>>>> sudo -s <<'EOSUDO'
>>>>> + test -e "${ROOTFSDIR}/chroot-setup.sh" && \
>>>>> + "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
>>>>> + rm -f "${ROOTFSDIR}/chroot-setup.sh"
>>>>> +
>>>>> test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
>>>>> find "${ROOTFSDIR}/usr/bin" \
>>>>> -maxdepth 1 -name 'qemu-*-static' -type f -delete
>>>>> @@ -42,6 +206,12 @@ EOSUDO
>>>>> do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>>>>> python do_rootfs_postprocess() {
>>>>> + # Take care that its correctly mounted:
>>>>> + bb.build.exec_func('rootfs_do_mounts', d)
>>>>> + # Take care that qemu-*-static is available, since it could
>>>>> have been
>>>>> + # removed on a previous execution of this task:
>>>>> + bb.build.exec_func('rootfs_do_qemu', d)
>>>>> +
>>>>> cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
>>>>> if cmds is None or not cmds.strip():
>>>>> return
>>>>> @@ -49,7 +219,7 @@ python do_rootfs_postprocess() {
>>>>> for cmd in cmds:
>>>>> bb.build.exec_func(cmd, d)
>>>>> }
>>>>> -addtask rootfs_postprocess before do_rootfs after do_rootfs_install
>>>>> +addtask rootfs_postprocess before do_rootfs
>>>>> python do_rootfs() {
>>>>> """Virtual task"""
>>>>> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> index b740c57..f8741c2 100644
>>>>> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
>>>>> @@ -327,6 +327,11 @@ isar_bootstrap() {
>>>>> mount --make-rslave ${ROOTFSDIR}/sys
>>>>> export DEBIAN_FRONTEND=noninteractive
>>>>> +
>>>>> + if [ ${IS_HOST} ]; then
>>>>> + chroot "${ROOTFSDIR}" /usr/bin/dpkg
>>>>> --add-architecture ${DISTRO_ARCH}
>>>>> + fi
>>>>> +
>>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
>>>>> chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
>>>>> -o Debug::pkgProblemResolver=yes
>>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> index 73cd548..121992c 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>>> require buildchroot.inc
>>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>>> BUILDCHROOT_PREINSTALL ?= "make \
>>>>> debhelper \
>>>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>>>> automake \
>>>>> devscripts \
>>>>> equivs \
>>>>> - libc6:${DISTRO_ARCH}"
>>>>> -
>>>>> -# According to the wiki page:
>>>>> -# https://wiki.debian.org/CrossToolchains
>>>>> -BUILDCHROOT_PREINSTALL_append_armhf +=
>>>>> "binutils-arm-linux-gnueabihf \
>>>>> - crossbuild-essential-armhf"
>>>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>>>> - crossbuild-essential-arm64"
>>>>> -
>>>>> -
>>>>> -PARAMS = "--host-arch '${HOST_ARCH}' --host-distro '${HOST_DISTRO}'"
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-host:do_bootstrap"
>>>>> + libc6:${DISTRO_ARCH} \
>>>>> + crossbuild-essential-${DISTRO_ARCH}"
>>>>> diff --git
>>>>> a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> index 20f4d23..5a01258 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-target.bb
>>>>> @@ -21,5 +21,3 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
>>>>> devscripts \
>>>>> equivs \
>>>>> adduser"
>>>>> -
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-target:do_bootstrap"
>>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> index ca9ce51..40fc8b0 100644
>>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
>>>>> @@ -13,18 +13,25 @@ SRC_URI = "file://configscript.sh \
>>>>> file://deps.sh"
>>>>> PV = "1.0"
>>>>> -inherit isar-bootstrap-helper
>>>>> +inherit rootfs
>>>>> BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
>>>>> +ROOTFSDIR = "${BUILDCHROOT_DIR}"
>>>>> +ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
>>>>> +ROOTFS_CLEAN_FILES = ""
>>>>> +
>>>>> +rootfs_do_mounts_append() {
>>>>> + sudo -s <<'EOSUDO'
>>>>> + mkdir -p '${BUILDCHROOT_DIR}/downloads'
>>>>> + mountpoint -q '${BUILDCHROOT_DIR}/downloads' || \
>>>>> + mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
>>>>> +EOSUDO
>>>>> +}
>>>>> -do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
>>>>> - ${BUILDCHROOT_DIR}/isar-apt \
>>>>> - ${BUILDCHROOT_DIR}/downloads \
>>>>> - ${BUILDCHROOT_DIR}/home/builder"
>>>>> -
>>>>> -do_build() {
>>>>> - setup_root_file_system ${PARAMS} ${BUILDCHROOT_DIR}
>>>>> ${BUILDCHROOT_PREINSTALL}
>>>>> -
>>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "buildchroot_install_files"
>>>>> +buildchroot_install_files() {
>>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/home/builder"
>>>>> + sudo mkdir -p "${BUILDCHROOT_DIR}/isar-apt"
>>>>> # Install package builder script
>>>>> sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
>>>>> sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
>>>>> @@ -38,5 +45,4 @@ do_build() {
>>>>> GROUP_ID=$(id -g)
>>>>> sudo chroot ${BUILDCHROOT_DIR} /configscript.sh $USER_ID
>>>>> $GROUP_ID
>>>>> - sudo mount --bind ${DL_DIR} ${BUILDCHROOT_DIR}/downloads
>>>>> }
>>>>> diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> index 49aeb47..57d52ba 100644
>>>>> --- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> +++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
>>>>> @@ -12,10 +12,25 @@ SRC_URI = " \
>>>>> file://configscript.sh \
>>>>> file://README.sdk"
>>>>> PV = "0.1"
>>>>> -
>>>>> -inherit isar-bootstrap-helper
>>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>>> +TOOLCHAIN = "crossbuild-essential-${DISTRO_ARCH}"
>>>>> +TOOLCHAIN_${HOST_ARCH} = "build-essential"
>>>>> +TOOLCHAIN_i386 = "build-essential"
>>>>> +
>>>>> +inherit rootfs
>>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>>> +ROOTFSDIR = "${S}"
>>>>> +ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
>>>>> +ROOTFS_FEATURES += "copy-package-cache"
>>>>> +
>>>>> +python() {
>>>>> + if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
>>>>> + raise bb.parse.SkipRecipe("SDK doesn't support {} as
>>>>> host".format(
>>>>> + d.getVar("ROOTFS_ARCH")))
>>>>> +}
>>>>> +
>>>>> SDKCHROOT_PREINSTALL := "debhelper \
>>>>> autotools-dev \
>>>>> dpkg \
>>>>> @@ -28,29 +43,14 @@ SDKCHROOT_PREINSTALL := "debhelper \
>>>>> S = "${WORKDIR}/rootfs"
>>>>> -do_build[dirs] = "${DEPLOY_DIR_IMAGE}"
>>>>> -do_build[root_cleandirs] = "${S} \
>>>>> - ${S}/isar-apt"
>>>>> -
>>>>> -do_build[depends] = "isar-apt:do_cache_config
>>>>> isar-bootstrap-host:do_bootstrap"
>>>>> -
>>>>> -do_build() {
>>>>> -
>>>>> - if [ ${HOST_DISTRO} != "debian-stretch" ]; then
>>>>> - bbfatal "SDK doesn't support ${HOST_DISTRO}"
>>>>> - fi
>>>>> - if [ ${HOST_ARCH} != "i386" -a ${HOST_ARCH} != "amd64" ]; then
>>>>> - bbfatal "SDK doesn't support ${HOST_ARCH} as host"
>>>>> - fi
>>>>> -
>>>>> - if [ ${HOST_ARCH} = ${DISTRO_ARCH} -o ${DISTRO_ARCH} = "i386"
>>>>> ]; then
>>>>> - packages="${SDKCHROOT_PREINSTALL} build-essential"
>>>>> - else
>>>>> - packages="${SDKCHROOT_PREINSTALL}
>>>>> crossbuild-essential-${DISTRO_ARCH}"
>>>>> - fi
>>>>> -
>>>>> - setup_root_file_system --copyisarapt --host-arch
>>>>> '${HOST_ARCH}' --host-distro '${HOST_DISTRO}' "${S}" $packages
>>>>> +ROOTFS_CONFIGURE_COMMAND += "rootfs_configure_isar_apt_dir"
>>>>> +rootfs_configure_isar_apt_dir() {
>>>>> + # Copy isar-apt instead of mounting:
>>>>> + sudo cp -Trpfx ${REPO_ISAR_DIR}/${DISTRO} ${ROOTFSDIR}/isar-apt
>>>>> +}
>>>>> +ROOTFS_POSTPROCESS_COMMAND =+ "sdkchroot_install_files"
>>>>> +sdkchroot_install_files() {
>>>>> # Configure root filesystem
>>>>> sudo install -m 644 ${WORKDIR}/README.sdk ${S}
>>>>> sudo install -m 755 ${WORKDIR}/configscript.sh ${S}
>>>>>
>>>>
>>>>
>>>
>>
>>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH] rootfs.bbclass: add comment about task weights
2019-05-27 9:49 ` Maxim Yu. Osipov
@ 2019-05-27 10:44 ` claudius.heine.ext
2019-05-29 12:38 ` Maxim Yu. Osipov
0 siblings, 1 reply; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-27 10:44 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Signed-off-by: Claudius Heine <ch@denx.de>
---
Hi Maxim,
you can apply this patch after the patchset, or just merge it. However you like.
regards,
Claudius
---
meta/classes/rootfs.bbclass | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 5736eee..c7e0435 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -142,6 +142,10 @@ python do_rootfs_install() {
# 'isar-apt' (sdkchroot):
cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
+ # NOTE: The weights specify how long each task takes in seconds and are used
+ # by the MultiStageProgressReporter to render a progress bar for this task.
+ # To printout the measured weights on a run, add `debug=True` as a parameter
+ # the MultiStageProgressReporter constructor.
stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
for i in cmds]
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH] rootfs.bbclass: add comment about task weights
2019-05-27 10:44 ` [PATCH] rootfs.bbclass: add comment about task weights claudius.heine.ext
@ 2019-05-29 12:38 ` Maxim Yu. Osipov
0 siblings, 0 replies; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-29 12:38 UTC (permalink / raw)
To: claudius.heine.ext, isar-users; +Cc: Claudius Heine
On 5/27/19 12:44 PM, claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> Signed-off-by: Claudius Heine <ch@denx.de>
Applied to the 'next',
Regards,
Maxim.
> ---
>
> Hi Maxim,
>
> you can apply this patch after the patchset, or just merge it. However you like.
>
> regards,
> Claudius
>
> ---
> meta/classes/rootfs.bbclass | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 5736eee..c7e0435 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -142,6 +142,10 @@ python do_rootfs_install() {
> # 'isar-apt' (sdkchroot):
> cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
>
> + # NOTE: The weights specify how long each task takes in seconds and are used
> + # by the MultiStageProgressReporter to render a progress bar for this task.
> + # To printout the measured weights on a run, add `debug=True` as a parameter
> + # the MultiStageProgressReporter constructor.
> stage_weights = [int(d.getVarFlag(i, 'weight', True) or "20")
> for i in cmds]
>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-05-23 14:55 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline claudius.heine.ext
2019-05-24 12:49 ` Maxim Yu. Osipov
@ 2019-08-14 16:00 ` Jan Kiszka
2019-08-19 6:59 ` Claudius Heine
1 sibling, 1 reply; 30+ messages in thread
From: Jan Kiszka @ 2019-08-14 16:00 UTC (permalink / raw)
To: [ext] claudius.heine.ext@siemens.com, isar-users; +Cc: Claudius Heine
On 23.05.19 16:55, [ext] claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> This removes the isar-bootstrap-helper.bbclass and integrates its
> functionality into the rootfs.bbclass.
>
> It introduces the rootfs_install task that runs through multiple
> functions from command lists in order to allow easy integration of
> custom functions.
>
> The introduced functions are, in general order of execution:
>
> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
> rootfs_configure_*: basic configuration of the root file system,
> before any custom packages where installed. (`/dev`, `/proc`, etc.
> are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
> rootfs_do_mount: mount standard directories and repositories into the
> rootfs
> do_rootfs_install_*: downloading and installing packages. Functions
> are added to `ROOTFS_INSTALL_COMMAND`.
>
> Signed-off-by: Claudius Heine <ch@denx.de>
> ---
> meta/classes/image.bbclass | 24 +--
> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
> .../isar-bootstrap/isar-bootstrap.inc | 5 +
> .../buildchroot/buildchroot-host.bb | 16 +-
> .../buildchroot/buildchroot-target.bb | 2 -
> .../buildchroot/buildchroot.inc | 26 ++-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
> 8 files changed, 228 insertions(+), 197 deletions(-)
> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>
[...]
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> index 73cd548..121992c 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>
> require buildchroot.inc
> +ROOTFS_ARCH = "${HOST_ARCH}"
> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>
> BUILDCHROOT_PREINSTALL ?= "make \
> debhelper \
> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
> automake \
> devscripts \
> equivs \
> - libc6:${DISTRO_ARCH}"
> -
> -# According to the wiki page:
> -# https://wiki.debian.org/CrossToolchains
> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
> - crossbuild-essential-armhf"
> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
> - crossbuild-essential-arm64"
Where did binutils go after this patch? Don't we need them? Wondering because
there is no remark in the commit why they were removed.
Jan
--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-08-14 16:00 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline Jan Kiszka
@ 2019-08-19 6:59 ` Claudius Heine
2019-08-19 7:02 ` Jan Kiszka
0 siblings, 1 reply; 30+ messages in thread
From: Claudius Heine @ 2019-08-19 6:59 UTC (permalink / raw)
To: Jan Kiszka, isar-users; +Cc: Claudius Heine
Hi Jan,
On 14/08/2019 18.00, Jan Kiszka wrote:
> On 23.05.19 16:55, [ext] claudius.heine.ext@siemens.com wrote:
>> From: Claudius Heine <ch@denx.de>
>>
>> This removes the isar-bootstrap-helper.bbclass and integrates its
>> functionality into the rootfs.bbclass.
>>
>> It introduces the rootfs_install task that runs through multiple
>> functions from command lists in order to allow easy integration of
>> custom functions.
>>
>> The introduced functions are, in general order of execution:
>>
>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>> rootfs_configure_*: basic configuration of the root file system,
>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>> are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
>> rootfs_do_mount: mount standard directories and repositories into the
>> rootfs
>> do_rootfs_install_*: downloading and installing packages. Functions
>> are added to `ROOTFS_INSTALL_COMMAND`.
>>
>> Signed-off-by: Claudius Heine <ch@denx.de>
>> ---
>> meta/classes/image.bbclass | 24 +--
>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>> .../buildchroot/buildchroot-host.bb | 16 +-
>> .../buildchroot/buildchroot-target.bb | 2 -
>> .../buildchroot/buildchroot.inc | 26 ++-
>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>> 8 files changed, 228 insertions(+), 197 deletions(-)
>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>
>
> [...]
>
>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> index 73cd548..121992c 100644
>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>> require buildchroot.inc
>> +ROOTFS_ARCH = "${HOST_ARCH}"
>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>> BUILDCHROOT_PREINSTALL ?= "make \
>> debhelper \
>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>> automake \
>> devscripts \
>> equivs \
>> - libc6:${DISTRO_ARCH}"
>> -
>> -# According to the wiki page:
>> -# https://wiki.debian.org/CrossToolchains
>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>> - crossbuild-essential-armhf"
>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>> - crossbuild-essential-arm64"
>
> Where did binutils go after this patch? Don't we need them? Wondering
> because there is no remark in the commit why they were removed.
Well crossbuild-essential-* is added and just like 'build-essential'
binutils is a dependency from it:
crossbuild-essential-* > gcc-*-linux-gnueabi > gcc-8-*-linux-gnueabi >
binutils-*-linux-gnueabi
So that was just a removal of unneeded package names, that get installed
anyway.
Claudius
>
> Jan
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-08-19 6:59 ` Claudius Heine
@ 2019-08-19 7:02 ` Jan Kiszka
2019-08-19 7:09 ` Claudius Heine
0 siblings, 1 reply; 30+ messages in thread
From: Jan Kiszka @ 2019-08-19 7:02 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
On 19.08.19 08:59, Claudius Heine wrote:
> Hi Jan,
>
> On 14/08/2019 18.00, Jan Kiszka wrote:
>> On 23.05.19 16:55, [ext] claudius.heine.ext@siemens.com wrote:
>>> From: Claudius Heine <ch@denx.de>
>>>
>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>> functionality into the rootfs.bbclass.
>>>
>>> It introduces the rootfs_install task that runs through multiple
>>> functions from command lists in order to allow easy integration of
>>> custom functions.
>>>
>>> The introduced functions are, in general order of execution:
>>>
>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>> rootfs_configure_*: basic configuration of the root file system,
>>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>>> are not mounted). Functions are added to `ROOTFS_CONFIGURE_COMMAND`.
>>> rootfs_do_mount: mount standard directories and repositories into the
>>> rootfs
>>> do_rootfs_install_*: downloading and installing packages. Functions
>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>
>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>> ---
>>> meta/classes/image.bbclass | 24 +--
>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>> meta/classes/rootfs.bbclass | 172 +++++++++++++++++-
>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>> .../buildchroot/buildchroot-target.bb | 2 -
>>> .../buildchroot/buildchroot.inc | 26 ++-
>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>
>>
>> [...]
>>
>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> index 73cd548..121992c 100644
>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>> require buildchroot.inc
>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>> BUILDCHROOT_PREINSTALL ?= "make \
>>> debhelper \
>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>> automake \
>>> devscripts \
>>> equivs \
>>> - libc6:${DISTRO_ARCH}"
>>> -
>>> -# According to the wiki page:
>>> -# https://wiki.debian.org/CrossToolchains
>>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>>> - crossbuild-essential-armhf"
>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>> - crossbuild-essential-arm64"
>>
>> Where did binutils go after this patch? Don't we need them? Wondering because
>> there is no remark in the commit why they were removed.
>
> Well crossbuild-essential-* is added and just like 'build-essential' binutils is
> a dependency from it:
>
> crossbuild-essential-* > gcc-*-linux-gnueabi > gcc-8-*-linux-gnueabi >
> binutils-*-linux-gnueabi
>
> So that was just a removal of unneeded package names, that get installed anyway.
That is what was missing in the commit...
Jan
--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline
2019-08-19 7:02 ` Jan Kiszka
@ 2019-08-19 7:09 ` Claudius Heine
0 siblings, 0 replies; 30+ messages in thread
From: Claudius Heine @ 2019-08-19 7:09 UTC (permalink / raw)
To: Jan Kiszka, isar-users; +Cc: Claudius Heine
On 19/08/2019 09.02, Jan Kiszka wrote:
> On 19.08.19 08:59, Claudius Heine wrote:
>> Hi Jan,
>>
>> On 14/08/2019 18.00, Jan Kiszka wrote:
>>> On 23.05.19 16:55, [ext] claudius.heine.ext@siemens.com wrote:
>>>> From: Claudius Heine <ch@denx.de>
>>>>
>>>> This removes the isar-bootstrap-helper.bbclass and integrates its
>>>> functionality into the rootfs.bbclass.
>>>>
>>>> It introduces the rootfs_install task that runs through multiple
>>>> functions from command lists in order to allow easy integration of
>>>> custom functions.
>>>>
>>>> The introduced functions are, in general order of execution:
>>>>
>>>> rootfs_prepare: copy the isar-bootstrap rootfs base to workdir
>>>> rootfs_configure_*: basic configuration of the root file system,
>>>> before any custom packages where installed. (`/dev`, `/proc`, etc.
>>>> are not mounted). Functions are added to
>>>> `ROOTFS_CONFIGURE_COMMAND`.
>>>> rootfs_do_mount: mount standard directories and repositories into
>>>> the
>>>> rootfs
>>>> do_rootfs_install_*: downloading and installing packages. Functions
>>>> are added to `ROOTFS_INSTALL_COMMAND`.
>>>>
>>>> Signed-off-by: Claudius Heine <ch@denx.de>
>>>> ---
>>>> meta/classes/image.bbclass | 24 +--
>>>> meta/classes/isar-bootstrap-helper.bbclass | 132 --------------
>>>> meta/classes/rootfs.bbclass | 172
>>>> +++++++++++++++++-
>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>> .../buildchroot/buildchroot.inc | 26 ++-
>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++---
>>>> 8 files changed, 228 insertions(+), 197 deletions(-)
>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>
>>>
>>> [...]
>>>
>>>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> index 73cd548..121992c 100644
>>>> --- a/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> +++ b/meta/recipes-devtools/buildchroot/buildchroot-host.bb
>>>> @@ -7,6 +7,8 @@ DESCRIPTION = "Isar development filesystem for host"
>>>> PF = "${PN}-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}"
>>>> require buildchroot.inc
>>>> +ROOTFS_ARCH = "${HOST_ARCH}"
>>>> +ROOTFS_DISTRO = "${HOST_DISTRO}"
>>>> BUILDCHROOT_PREINSTALL ?= "make \
>>>> debhelper \
>>>> @@ -18,15 +20,5 @@ BUILDCHROOT_PREINSTALL ?= "make \
>>>> automake \
>>>> devscripts \
>>>> equivs \
>>>> - libc6:${DISTRO_ARCH}"
>>>> -
>>>> -# According to the wiki page:
>>>> -# https://wiki.debian.org/CrossToolchains
>>>> -BUILDCHROOT_PREINSTALL_append_armhf += "binutils-arm-linux-gnueabihf \
>>>> - crossbuild-essential-armhf"
>>>> -BUILDCHROOT_PREINSTALL_append_arm64 += "binutils-aarch64-linux-gnu \
>>>> - crossbuild-essential-arm64"
>>>
>>> Where did binutils go after this patch? Don't we need them? Wondering
>>> because there is no remark in the commit why they were removed.
>>
>> Well crossbuild-essential-* is added and just like 'build-essential'
>> binutils is a dependency from it:
>>
>> crossbuild-essential-* > gcc-*-linux-gnueabi > gcc-8-*-linux-gnueabi >
>> binutils-*-linux-gnueabi
>>
>> So that was just a removal of unneeded package names, that get
>> installed anyway.
>
> That is what was missing in the commit...
Well for me that was just some minor refactor on the side. But you are
right, it should have probably been an additional commit because its an
unrelated change. Sometimes it is difficult to separate each independent
change from each other, when a lot of big changes where done all over.
Claudius
>
> Jan
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread
* [PATCH v4 3/8] meta/classes: add image-locales-extension class
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 1/8] meta: remove transient package support claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 2/8] split up isar-bootstrap helper and implement pre-process pipeline claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 4/8] meta/classes: add image-account-extension class claudius.heine.ext
` (6 subsequent siblings)
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
This class allows to configure the locales of the image and also uses
'localepurge' to remove unwanted locales.
The locale can be configured the same way as with the
isar-cfg-localepurge package, via the `LOCALE_GEN` and `LOCALE_DEFAULT`
variables.
Signed-off-by: Claudius Heine <ch@denx.de>
---
meta/classes/image-locales-extension.bbclass | 97 ++++++++++++++++++++
meta/classes/image.bbclass | 1 +
meta/classes/rootfs.bbclass | 4 +
3 files changed, 102 insertions(+)
create mode 100644 meta/classes/image-locales-extension.bbclass
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
new file mode 100644
index 0000000..3c0758f
--- /dev/null
+++ b/meta/classes/image-locales-extension.bbclass
@@ -0,0 +1,97 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+#
+# This class extends the image.bbclass for setting locales and purging unneeded
+# ones.
+
+LOCALE_GEN ?= "en_US.UTF-8 UTF-8\n\
+ en_US ISO-8859-1\n"
+LOCALE_DEFAULT ?= "en_US.UTF-8"
+
+def get_locale_gen(d, sep='\n'):
+ locale_gen = d.getVar("LOCALE_GEN", True) or ""
+ return sep.join(sorted(set(i.strip()
+ for i in locale_gen.split('\\n')
+ if i.strip())))
+
+def get_nopurge(d):
+ locale_gen = d.getVar("LOCALE_GEN", True) or ""
+ return '\n'.join(sorted(set(i.strip()
+ for j in locale_gen.split('\\n')
+ if j.strip()
+ for i in (j.split()[0].split("_")[0],
+ j.split()[0].split(".")[0],
+ j.split()[0]))))
+
+ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+image_install_localepurge_download[weight] = "40"
+image_install_localepurge_download() {
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only localepurge
+}
+
+ROOTFS_INSTALL_COMMAND += "image_install_localepurge_install"
+image_install_localepurge_install[weight] = "700"
+image_install_localepurge_install() {
+
+ # Generate locale and localepurge configuration:
+ cat<<__EOF__ > ${WORKDIR}/locale.gen
+${@get_locale_gen(d)}
+__EOF__
+ cat<<__EOF__ > ${WORKDIR}/locale.debconf
+locales locales/locales_to_be_generated multiselect ${@get_locale_gen(d, ', ')}
+locales locales/default_environment_locale select ${LOCALE_DEFAULT}
+__EOF__
+ cat<<__EOF__ > ${WORKDIR}/locale.default
+LANG=${LOCALE_DEFAULT}
+__EOF__
+ cat<<__EOF__ > ${WORKDIR}/locale.nopurge
+#USE_DPKG
+MANDELETE
+DONTBOTHERNEWLOCALE
+#SHOWFREEDSPACE
+#QUICKNDIRTYCALC
+#VERBOSE
+${@get_nopurge(d)}
+__EOF__
+
+ # Install configuration into image:
+ sudo -E -s <<'EOSUDO'
+ cat '${WORKDIR}/locale.gen' >> '${ROOTFSDIR}/etc/locale.gen'
+ cat '${WORKDIR}/locale.default' > '${ROOTFSDIR}/etc/default/locale'
+ cat '${WORKDIR}/locale.nopurge' > '${ROOTFSDIR}/etc/locale.nopurge'
+ cat '${WORKDIR}/locale.debconf' > '${ROOTFSDIR}/tmp/locale.debconf'
+
+ # Enter image and trigger locales config and localepurge:
+ chroot '${ROOTFSDIR}' /bin/sh <<'EOSH'
+ localepurge_state='i'
+ if dpkg -s localepurge 2>/dev/null >&2
+ then
+ echo 'localepurge was installed (leaving it installed later)'
+ else
+ localepurge_state='p'
+ echo 'localepurge was not installed (removing it later)'
+ apt-get ${ROOTFS_APT_ARGS} localepurge
+ fi
+
+ echo 'running locale debconf-set-selections'
+ debconf-set-selections /tmp/locale.debconf
+ rm -f '/tmp/locale.debconf'
+
+ echo 'reconfigure locales'
+ dpkg-reconfigure -f noninteractive locales
+
+ echo 'running localepurge'
+ localepurge
+
+ if [ "$localepurge_state" = 'p' ]
+ then
+ echo removing localepurge...
+ apt-get purge --yes localepurge
+ apt-get autoremove --purge --yes
+ fi
+EOSH
+EOSUDO
+}
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 99eea92..dce6638 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,6 +60,7 @@ inherit image-sdk-extension
inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
+inherit image-locales-extension
# Extra space for rootfs in MB
ROOTFS_EXTRA ?= "64"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 0fc8ba8..5736eee 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -99,6 +99,7 @@ rootfs_install_pkgs_update() {
-o Dir::Etc::sourceparts="-" \
-o APT::Get::List-Cleanup="0"
}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_resolvconf"
rootfs_install_resolvconf[weight] = "1"
rootfs_install_resolvconf() {
@@ -112,6 +113,9 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}
+ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+
ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
rootfs_install_clean_files() {
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH v4 4/8] meta/classes: add image-account-extension class
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (2 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 3/8] meta/classes: add image-locales-extension class claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-31 7:29 ` Jan Kiszka
2019-05-23 14:55 ` [PATCH v4 5/8] doc: update description of image customization claudius.heine.ext
` (5 subsequent siblings)
9 siblings, 1 reply; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
This class allows to configure user and group accounts of the image.
Groups or users that should be configured/created are added into the
`GROUPS` or `USERS` variable.
The configuration itself is then added to each groups or users
`GROUP_<groupname>` or `USER_<username>` flags.
The flags available for groups are `gid` and `flags`. The `flags`
variable contains some additional options for the group. With this patch
only `system` is supported for groups, allowing to create groups with
`groupadd` with the `--system` parameter.
The flags available for users are `password`, `expire`, `inactive`,
`uid`, `gid`, `comment`, `home`, `shell`, `groups` and `flags`. The
additional flags for users are `no-create-home`, `create-home`, `system`
and `allow-empty-password`.
Signed-off-by: Claudius Heine <ch@denx.de>
---
meta/classes/image-account-extension.bbclass | 257 +++++++++++++++++++
meta/classes/image.bbclass | 1 +
2 files changed, 258 insertions(+)
create mode 100644 meta/classes/image-account-extension.bbclass
diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
new file mode 100644
index 0000000..22754da
--- /dev/null
+++ b/meta/classes/image-account-extension.bbclass
@@ -0,0 +1,257 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+#
+# This class extends the image.bbclass for creating user accounts and groups.
+
+USERS ??= ""
+
+#USERS += "root"
+#USER_root[password] = "" # Encrypted password
+#USER_root[expire] = ""
+#USER_root[inactive] = ""
+#USER_root[uid] = ""
+#USER_root[gid] = "" # If first character is a number: gid, otherwise groupname
+#USER_root[comment] = "The ultimate root user"
+#USER_root[home] = "/home/root"
+#USER_root[shell] = "/bin/sh"
+#USER_root[groups] = "audio video"
+#USER_root[flags] = "no-create-home create-home system allow-empty-password"
+
+GROUPS ??= ""
+
+#GROUPS += "root"
+#GROUP_root[gid] = ""
+#GROUP_root[flags] = "system"
+
+def gen_accounts_array(d, listname, entryname, flags, verb_flags=None):
+ from itertools import chain
+
+ entries = (d.getVar(listname, True) or "").split()
+ return " ".join(
+ ":".join(
+ chain(
+ (entry,),
+ (
+ (",".join(
+ (
+ d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+ ).split()
+ ) if flag not in (verb_flags or []) else (
+ d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+ )).replace(":","=")
+ for flag in flags
+ ),
+ )
+ )
+ for entry in entries
+ )
+
+# List of space separated entries, where each entry has the format:
+# username:encryptedpassword:expiredate:inactivenumber:userid:groupid:comment:homedir:shell:group1,group2:flag1,flag2
+IMAGE_ACCOUNTS_USERS =+ "${@gen_accounts_array(d, 'USERS', 'USER', ['password', 'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
+
+# List of space separated entries, where each entry has the format:
+# groupname:groupid:flag1,flag2
+IMAGE_ACCOUNTS_GROUPS =+ "${@gen_accounts_array(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
+
+ROOTFS_CONFIGURE_COMMAND += "image_configure_accounts"
+image_configure_accounts[weight] = "3"
+image_configure_accounts() {
+ # Create groups
+ # Add space to the end of the list:
+ list='${@" ".join(d.getVar('IMAGE_ACCOUNTS_GROUPS', True).split())} '
+ while true; do
+ # Pop first group entry:
+ list_rest="${list#*:*:* }"
+ entry="${list%%${list_rest}}"
+ list="${list_rest}"
+
+ if [ -z "${entry}" ]; then
+ break
+ fi
+
+ # Add colon to the end of the entry and remove trailing space:
+ entry="${entry% }:"
+
+ # Decode entries:
+ name="${entry%%:*}"
+ entry="${entry#${name}:}"
+
+ gid="${entry%%:*}"
+ entry="${entry#${gid}:}"
+
+ flags="${entry%%:*}"
+ entry="${entry#${flags}:}"
+
+ flags=",${flags}," # Needed for searching for substrings
+
+ # Check if user already exists:
+ if grep -q "^${name}:" '${ROOTFSDIR}/etc/group'; then
+ exists="y"
+ else
+ exists="n"
+ fi
+
+ # Create arguments:
+ set -- # clear arguments
+
+ if [ -n "$gid" ]; then
+ set -- "$@" --gid "$gid"
+ fi
+
+ if [ "n" = "$exists" ]; then
+ if [ "${flags}" != "${flags%*,system,*}" ]; then
+ set -- "$@" --system
+ fi
+ fi
+
+ # Create or modify groups:
+ if [ "y" = "$exists" ]; then
+ if [ -z "$@" ]; then
+ echo "Do not execute groupmod (no changes)."
+ else
+ echo "Execute groupmod with \"$@\" for \"$name\""
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/sbin/groupmod "$@" "$name"
+ fi
+ else
+ echo "Execute groupadd with \"$@\" for \"$name\""
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/sbin/groupadd "$@" "$name"
+ fi
+ done
+
+ # Create users
+ list='${@" ".join(d.getVar('IMAGE_ACCOUNTS_USERS', True).split())} '
+ while true; do
+ # Pop first user entry:
+ list_rest="${list#*:*:*:*:*:*:*:*:*:*:* }"
+ entry="${list%%${list_rest}}"
+ list="${list_rest}"
+
+ if [ -z "${entry}" ]; then
+ break
+ fi
+
+ # Add colon to the end of the entry and remove trailing space:
+ entry="${entry% }:"
+
+ # Decode entries:
+ name="${entry%%:*}"
+ entry="${entry#${name}:}"
+
+ password="${entry%%:*}"
+ entry="${entry#${password}:}"
+
+ expire="${entry%%:*}"
+ entry="${entry#${expire}:}"
+
+ inactive="${entry%%:*}"
+ entry="${entry#${inactive}:}"
+
+ uid="${entry%%:*}"
+ entry="${entry#${uid}:}"
+
+ gid="${entry%%:*}"
+ entry="${entry#${gid}:}"
+
+ comment="${entry%%:*}"
+ entry="${entry#${comment}:}"
+
+ home="${entry%%:*}"
+ entry="${entry#${home}:}"
+
+ shell="${entry%%:*}"
+ entry="${entry#${shell}:}"
+
+ groups="${entry%%:*}"
+ entry="${entry#${groups}:}"
+
+ flags="${entry%%:*}"
+ entry="${entry#${flags}:}"
+
+ flags=",${flags}," # Needed for searching for substrings
+
+ # Check if user already exists:
+ if grep -q "^${name}:" '${ROOTFSDIR}/etc/passwd'; then
+ exists="y"
+ else
+ exists="n"
+ fi
+
+ # Create arguments:
+ set -- # clear arguments
+
+ if [ -n "$expire" ]; then
+ set -- "$@" --expiredate "$expire"
+ fi
+
+ if [ -n "$inactive" ]; then
+ set -- "$@" --inactive "$inactive"
+ fi
+
+ if [ -n "$uid" ]; then
+ set -- "$@" --uid "$uid"
+ fi
+
+ if [ -n "$gid" ]; then
+ set -- "$@" --gid "$gid"
+ fi
+
+ if [ -n "$comment" ]; then
+ set -- "$@" --comment "$comment"
+ fi
+
+ if [ -n "$home" ]; then
+ if [ "y" = "$exists" ]; then
+ set -- "$@" --home "$home" --move-home
+ else
+ set -- "$@" --home-dir "$home"
+ fi
+ fi
+
+ if [ -n "$shell" ]; then
+ set -- "$@" --shell "$shell"
+ fi
+
+ if [ -n "$groups" ]; then
+ set -- "$@" --groups "$groups"
+ fi
+
+ if [ "n" = "$exists" ]; then
+ if [ "${flags}" != "${flags%*,system,*}" ]; then
+ set -- "$@" --system
+ fi
+ if [ "${flags}" != "${flags%*,no-create-home,*}" ]; then
+ set -- "$@" --no-create-home
+ else
+ if [ "${flags}" != "${flags%*,create-home,*}" ]; then
+ set -- "$@" --create-home
+ fi
+ fi
+ fi
+
+ # Create or modify users:
+ if [ "y" = "$exists" ]; then
+ if [ -z "$@" ]; then
+ echo "Do not execute usermod (no changes)."
+ else
+ echo "Execute usermod with \"$@\" for \"$name\""
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/sbin/usermod "$@" "$name"
+ fi
+ else
+ echo "Execute useradd with \"$@\" for \"$name\""
+ sudo -E chroot '${ROOTFSDIR}' \
+ /usr/sbin/useradd "$@" "$name"
+ fi
+
+ # Set password:
+ if [ -n "$password" -o "${flags}" != "${flags%*,allow-empty-password,*}" ]; then
+ printf '%s:%s' "$name" "$password" | sudo chroot '${ROOTFSDIR}' \
+ /usr/sbin/chpasswd -e
+ fi
+ done
+}
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index dce6638..ef7983b 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -61,6 +61,7 @@ inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
+inherit image-account-extension
# Extra space for rootfs in MB
ROOTFS_EXTRA ?= "64"
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 4/8] meta/classes: add image-account-extension class
2019-05-23 14:55 ` [PATCH v4 4/8] meta/classes: add image-account-extension class claudius.heine.ext
@ 2019-05-31 7:29 ` Jan Kiszka
2019-06-03 9:14 ` Claudius Heine
0 siblings, 1 reply; 30+ messages in thread
From: Jan Kiszka @ 2019-05-31 7:29 UTC (permalink / raw)
To: [ext] claudius.heine.ext@siemens.com, isar-users; +Cc: Claudius Heine
On 23.05.19 16:55, [ext] claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> This class allows to configure user and group accounts of the image.
>
> Groups or users that should be configured/created are added into the
> `GROUPS` or `USERS` variable.
>
> The configuration itself is then added to each groups or users
> `GROUP_<groupname>` or `USER_<username>` flags.
>
> The flags available for groups are `gid` and `flags`. The `flags`
> variable contains some additional options for the group. With this patch
> only `system` is supported for groups, allowing to create groups with
> `groupadd` with the `--system` parameter.
>
> The flags available for users are `password`, `expire`, `inactive`,
> `uid`, `gid`, `comment`, `home`, `shell`, `groups` and `flags`. The
> additional flags for users are `no-create-home`, `create-home`, `system`
> and `allow-empty-password`.
>
> Signed-off-by: Claudius Heine <ch@denx.de>
> ---
> meta/classes/image-account-extension.bbclass | 257 +++++++++++++++++++
> meta/classes/image.bbclass | 1 +
> 2 files changed, 258 insertions(+)
> create mode 100644 meta/classes/image-account-extension.bbclass
>
> diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
> new file mode 100644
> index 0000000..22754da
> --- /dev/null
> +++ b/meta/classes/image-account-extension.bbclass
> @@ -0,0 +1,257 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +# This class extends the image.bbclass for creating user accounts and groups.
> +
> +USERS ??= ""
> +
> +#USERS += "root"
> +#USER_root[password] = "" # Encrypted password
> +#USER_root[expire] = ""
> +#USER_root[inactive] = ""
> +#USER_root[uid] = ""
> +#USER_root[gid] = "" # If first character is a number: gid, otherwise groupname
> +#USER_root[comment] = "The ultimate root user"
> +#USER_root[home] = "/home/root"
> +#USER_root[shell] = "/bin/sh"
> +#USER_root[groups] = "audio video"
> +#USER_root[flags] = "no-create-home create-home system allow-empty-password"
> +
> +GROUPS ??= ""
> +
> +#GROUPS += "root"
> +#GROUP_root[gid] = ""
> +#GROUP_root[flags] = "system"
> +
> +def gen_accounts_array(d, listname, entryname, flags, verb_flags=None):
> + from itertools import chain
> +
> + entries = (d.getVar(listname, True) or "").split()
> + return " ".join(
> + ":".join(
> + chain(
> + (entry,),
> + (
> + (",".join(
> + (
> + d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> + ).split()
> + ) if flag not in (verb_flags or []) else (
> + d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> + )).replace(":","=")
> + for flag in flags
> + ),
> + )
> + )
> + for entry in entries
> + )
> +
> +# List of space separated entries, where each entry has the format:
> +# username:encryptedpassword:expiredate:inactivenumber:userid:groupid:comment:homedir:shell:group1,group2:flag1,flag2
> +IMAGE_ACCOUNTS_USERS =+ "${@gen_accounts_array(d, 'USERS', 'USER', ['password', 'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
> +
> +# List of space separated entries, where each entry has the format:
> +# groupname:groupid:flag1,flag2
> +IMAGE_ACCOUNTS_GROUPS =+ "${@gen_accounts_array(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
> +
> +ROOTFS_CONFIGURE_COMMAND += "image_configure_accounts"
> +image_configure_accounts[weight] = "3"
> +image_configure_accounts() {
> + # Create groups
> + # Add space to the end of the list:
> + list='${@" ".join(d.getVar('IMAGE_ACCOUNTS_GROUPS', True).split())} '
> + while true; do
> + # Pop first group entry:
> + list_rest="${list#*:*:* }"
> + entry="${list%%${list_rest}}"
> + list="${list_rest}"
> +
> + if [ -z "${entry}" ]; then
> + break
> + fi
> +
> + # Add colon to the end of the entry and remove trailing space:
> + entry="${entry% }:"
> +
> + # Decode entries:
> + name="${entry%%:*}"
> + entry="${entry#${name}:}"
> +
> + gid="${entry%%:*}"
> + entry="${entry#${gid}:}"
> +
> + flags="${entry%%:*}"
> + entry="${entry#${flags}:}"
> +
> + flags=",${flags}," # Needed for searching for substrings
> +
> + # Check if user already exists:
> + if grep -q "^${name}:" '${ROOTFSDIR}/etc/group'; then
> + exists="y"
> + else
> + exists="n"
> + fi
> +
> + # Create arguments:
> + set -- # clear arguments
> +
> + if [ -n "$gid" ]; then
> + set -- "$@" --gid "$gid"
> + fi
> +
> + if [ "n" = "$exists" ]; then
> + if [ "${flags}" != "${flags%*,system,*}" ]; then
> + set -- "$@" --system
> + fi
> + fi
> +
> + # Create or modify groups:
> + if [ "y" = "$exists" ]; then
> + if [ -z "$@" ]; then
> + echo "Do not execute groupmod (no changes)."
> + else
> + echo "Execute groupmod with \"$@\" for \"$name\""
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/sbin/groupmod "$@" "$name"
> + fi
> + else
> + echo "Execute groupadd with \"$@\" for \"$name\""
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/sbin/groupadd "$@" "$name"
> + fi
> + done
> +
> + # Create users
> + list='${@" ".join(d.getVar('IMAGE_ACCOUNTS_USERS', True).split())} '
> + while true; do
> + # Pop first user entry:
> + list_rest="${list#*:*:*:*:*:*:*:*:*:*:* }"
> + entry="${list%%${list_rest}}"
> + list="${list_rest}"
> +
> + if [ -z "${entry}" ]; then
> + break
> + fi
> +
> + # Add colon to the end of the entry and remove trailing space:
> + entry="${entry% }:"
> +
> + # Decode entries:
> + name="${entry%%:*}"
> + entry="${entry#${name}:}"
> +
> + password="${entry%%:*}"
> + entry="${entry#${password}:}"
> +
> + expire="${entry%%:*}"
> + entry="${entry#${expire}:}"
> +
> + inactive="${entry%%:*}"
> + entry="${entry#${inactive}:}"
> +
> + uid="${entry%%:*}"
> + entry="${entry#${uid}:}"
> +
> + gid="${entry%%:*}"
> + entry="${entry#${gid}:}"
> +
> + comment="${entry%%:*}"
> + entry="${entry#${comment}:}"
> +
> + home="${entry%%:*}"
> + entry="${entry#${home}:}"
> +
> + shell="${entry%%:*}"
> + entry="${entry#${shell}:}"
> +
> + groups="${entry%%:*}"
> + entry="${entry#${groups}:}"
> +
> + flags="${entry%%:*}"
> + entry="${entry#${flags}:}"
> +
> + flags=",${flags}," # Needed for searching for substrings
> +
> + # Check if user already exists:
> + if grep -q "^${name}:" '${ROOTFSDIR}/etc/passwd'; then
> + exists="y"
> + else
> + exists="n"
> + fi
> +
> + # Create arguments:
> + set -- # clear arguments
> +
> + if [ -n "$expire" ]; then
> + set -- "$@" --expiredate "$expire"
> + fi
> +
> + if [ -n "$inactive" ]; then
> + set -- "$@" --inactive "$inactive"
> + fi
> +
> + if [ -n "$uid" ]; then
> + set -- "$@" --uid "$uid"
> + fi
> +
> + if [ -n "$gid" ]; then
> + set -- "$@" --gid "$gid"
> + fi
> +
> + if [ -n "$comment" ]; then
> + set -- "$@" --comment "$comment"
> + fi
> +
> + if [ -n "$home" ]; then
> + if [ "y" = "$exists" ]; then
> + set -- "$@" --home "$home" --move-home
> + else
> + set -- "$@" --home-dir "$home"
> + fi
> + fi
> +
> + if [ -n "$shell" ]; then
> + set -- "$@" --shell "$shell"
> + fi
> +
> + if [ -n "$groups" ]; then
> + set -- "$@" --groups "$groups"
> + fi
> +
> + if [ "n" = "$exists" ]; then
> + if [ "${flags}" != "${flags%*,system,*}" ]; then
> + set -- "$@" --system
> + fi
> + if [ "${flags}" != "${flags%*,no-create-home,*}" ]; then
> + set -- "$@" --no-create-home
> + else
> + if [ "${flags}" != "${flags%*,create-home,*}" ]; then
> + set -- "$@" --create-home
> + fi
> + fi
> + fi
> +
> + # Create or modify users:
> + if [ "y" = "$exists" ]; then
> + if [ -z "$@" ]; then
> + echo "Do not execute usermod (no changes)."
> + else
> + echo "Execute usermod with \"$@\" for \"$name\""
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/sbin/usermod "$@" "$name"
> + fi
> + else
> + echo "Execute useradd with \"$@\" for \"$name\""
> + sudo -E chroot '${ROOTFSDIR}' \
> + /usr/sbin/useradd "$@" "$name"
> + fi
> +
> + # Set password:
> + if [ -n "$password" -o "${flags}" != "${flags%*,allow-empty-password,*}" ]; then
> + printf '%s:%s' "$name" "$password" | sudo chroot '${ROOTFSDIR}' \
> + /usr/sbin/chpasswd -e
> + fi
> + done
> +}
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index dce6638..ef7983b 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -61,6 +61,7 @@ inherit image-cache-extension
> inherit image-tools-extension
> inherit image-postproc-extension
> inherit image-locales-extension
> +inherit image-account-extension
>
> # Extra space for rootfs in MB
> ROOTFS_EXTRA ?= "64"
>
Seems like we are not rebuilding the affected recipes when variables change.
That's at least true for the root password I just played with.
Fixable?
Jan
--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 4/8] meta/classes: add image-account-extension class
2019-05-31 7:29 ` Jan Kiszka
@ 2019-06-03 9:14 ` Claudius Heine
0 siblings, 0 replies; 30+ messages in thread
From: Claudius Heine @ 2019-06-03 9:14 UTC (permalink / raw)
To: Jan Kiszka, isar-users; +Cc: Claudius Heine
Hi Jan,
On 31/05/2019 09.29, Jan Kiszka wrote:
[...]
>> +inherit image-account-extension
>> # Extra space for rootfs in MB
>> ROOTFS_EXTRA ?= "64"
>>
>
> Seems like we are not rebuilding the affected recipes when variables
> change. That's at least true for the root password I just played with.
> Fixable?
Interesting. My intention was to implement it in a way to avoid this
issue, but I guess that needs some further investigation then.
Just to make clear how I thought it worked:
This line should add the all bitbake functions in
`ROOTFS_CONFIGURE_COMMAND` and `ROOTFS_INSTALL_COMMAND` to the `vardeps`
flag. Because bitbake functions are variables as well, that should mean
that `do_rootfs_install` depends on the content of those functions,
including all expanded bitbake variables (at least so I thought):
meta/classes/rootfs.bbclass:
do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
${ROOTFS_INSTALL_COMMAND}"
It would be good to investigate why that didn't work. A simple fix would
be to add `IMAGE_ACCOUNTS_USERS` and `IMAGE_ACCOUNTS_GROUPS` to that
flag as well. But it would be good to investigate first why that
approach did not work and possible fix that, because that mechanism is
used elsewhere as well.
regards,
Claudius
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread
* [PATCH v4 5/8] doc: update description of image customization
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (3 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 4/8] meta/classes: add image-account-extension class claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 6/8] doc: some fixes claudius.heine.ext
` (4 subsequent siblings)
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Signed-off-by: Claudius Heine <ch@denx.de>
---
doc/user_manual.md | 51 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 40 insertions(+), 11 deletions(-)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index dbbe6f6..4c9f4ae 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -314,16 +314,12 @@ Some other variables include:
- `IMAGE_INSTALL` - The list of custom packages to build and install to target image, please refer to relative chapter for more information.
- `BB_NUMBER_THREADS` - The number of `bitbake` jobs that can be run in parallel. Please set this option according your host CPU cores number.
- - `LOCALE_GEN` - A `\n` seperated list of `/etc/locale.gen` entries desired on the target.
- - `LOCALE_DEFAULT` - The default locale used for the `LANG` and `LANGUAGE` variable in `/etc/locale`.
- `HOST_DISTRO` - The distro to use for SDK root filesystem (so far limited only to `debian-stretch`). This variable is optional.
- `HOST_ARCH` - The Debian architecture of SDK root filesystem (e.g., `amd64`). By default set to current Debian host architecture. This variable is optional.
- `HOST_DISTRO_APT_SOURCES` - List of apt source files for SDK root filesystem. This variable is optional.
- `HOST_DISTRO_APT_PREFERENCES` - List of apt preference files for SDK root filesystem. This variable is optional.
- `DISTRO_APT_PREMIRRORS` - The preferred mirror (append it to the default URI in the format `ftp.debian.org my.preferred.mirror`. This variable is optional.
- `THIRD_PARTY_APT_KEYS` - List of gpg key URIs used to verify apt repos for apt installation after bootstrapping
- - `CFG_ROOT_PW` - The encrypted root password to be set. To encrypt password use `mkpasswd`. You find `mkpasswd` in the `whois` package of Debian. If the variable is empty, root login is passwordless.
- - `CFG_ROOT_LOCKED` - If set to `1` the root account will be locked.
---
@@ -496,17 +492,50 @@ Isar contains additional image type classes that can be used as reference:
## Customize and configure image
-Customization and configuration of an image should be done via packages, see below.
+Customization and configuration of an image can be done in two ways:
+
+ 1. Creating and adding a configuration package to `IMAGE_INSTALL`, or
+ 2. Changing the bitbake variables of the image recipe.
+
+In cases where configuration is not image specific, does not contain any secrets and can be shared between images, creating and adding a configuration package to `IMAGE_INSTALL` is the right option. This should be the case with most product specific configuration files.
+
+In cases where the configuration would contain secrets like user passwords, that would be world readable in `postinst`, etc. script files, some image extensions where created, that allow customization of those options from within the image recipe using bitbake variables. (e.g. user and group management and locale settings)
+
+### Locale configuration
+
+Two variables can be used to configure the locale installed on a image:
+
+ - `LOCALE_GEN` - A `\n` seperated list of `/etc/locale.gen` entries desired on the target.
+ - `LOCALE_DEFAULT` - The default locale used for the `LANG` and `LANGUAGE` variable in `/etc/locale`.
+
+### User and group configuration
+
+Groups can be created or modified using the `GROUPS` and `GROUP_<groupname>` variable or their flags.
+
+The `GROUPS` variable contains a space separated list of group names that should be modified or created. Each entry of this variable should have a corresponding `GROUP_<groupname>` variable.
-Adding those configuration packages to the image can be done in two ways:
+The `GROUP_<groupname>` variable contains the settings of a group named `groupname` in its flags. The following flags can be used:
- 1. Simply adding the package to `IMAGE_INSTALL`, like any other isar created package, or
- 2. Adding the package to `IMAGE_TRANSIENT_PACKAGES`.
+ - `gid` - The numeric group id.
+ - `flags` - A list of additional flags of the group. Those are the currently recognized flags:
+ - `system` - The group is created using the `--system` parameter.
-In most cases adding the configuration package to `IMAGE_INSTALL` is the right option.
+The `USERS` and `USER_<username>` variable works similar to the `GROUPS` and `GROUP_<groupname>` variable. The difference are the accepted flags of the `USER_<username>` variable. It accepts the following flags:
-In cases were the configuration script of the package has some external dependencies, that should not be part of the final image, then `IMAGE_TRANSIENT_PACKAGES` is the right option.
-Packages in the `IMAGE_TRANSIENT_PACKAGES` variable are installed to the image and purged in the next step. If such a configuration package deploys file as part of their content, then those files will be removed as well.
+ - `password` - The crypt(3) encrypted password. To encrypt a password use for example `mkpasswd` or `openssl passwd -6`. You can find `mkpasswd` in the `whois` package of Debian.
+ - `expire` - A `YYYY-MM-DD` formatted date on which the user account will be disabled. (see useradd(8))
+ - `inactive` - The number of days after a password expires until the account is permanently disabled. (see useradd(8))
+ - `uid` - The numeric user id.
+ - `gid` - The numeric group id or group name of this users initial login group.
+ - `comment` - This users comment field. Commonly the following format `full name,room number,work phone number,home phone number,other entry`.
+ - `home` - This users home directory
+ - `shell` - This users login shell
+ - `groups` - A space separated list of groups this user is a member of.
+ - `flags` - A list of additional flags of the user:
+ - `no-create-home` - `useradd` will be called with `-M` to prevent creation of the users home directory.
+ - `create-home` - `useradd` will be called with `-m` to force creation of the users home directory.
+ - `system` - `useradd` will be called with `--system`.
+ - `allow-empty-password` - Even if the `password` flag is empty, it will still be set. This results in a login without password.
---
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH v4 6/8] doc: some fixes
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (4 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 5/8] doc: update description of image customization claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 7/8] meta-isar: local.conf.sample: update root password and isar user creation claudius.heine.ext
` (3 subsequent siblings)
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Signed-off-by: Claudius Heine <ch@denx.de>
---
doc/user_manual.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index 4c9f4ae..ed181aa 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -314,7 +314,7 @@ Some other variables include:
- `IMAGE_INSTALL` - The list of custom packages to build and install to target image, please refer to relative chapter for more information.
- `BB_NUMBER_THREADS` - The number of `bitbake` jobs that can be run in parallel. Please set this option according your host CPU cores number.
- - `HOST_DISTRO` - The distro to use for SDK root filesystem (so far limited only to `debian-stretch`). This variable is optional.
+ - `HOST_DISTRO` - The distro to use for SDK root filesystem. This variable is optional.
- `HOST_ARCH` - The Debian architecture of SDK root filesystem (e.g., `amd64`). By default set to current Debian host architecture. This variable is optional.
- `HOST_DISTRO_APT_SOURCES` - List of apt source files for SDK root filesystem. This variable is optional.
- `HOST_DISTRO_APT_PREFERENCES` - List of apt preference files for SDK root filesystem. This variable is optional.
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH v4 7/8] meta-isar: local.conf.sample: update root password and isar user creation
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (5 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 6/8] doc: some fixes claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-23 14:55 ` [PATCH v4 8/8] RECIPE-API-CHANGELOG: update transient package removal + root password claudius.heine.ext
` (2 subsequent siblings)
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Signed-off-by: Claudius Heine <ch@denx.de>
---
meta-isar/conf/local.conf.sample | 12 +++++++++++-
meta-isar/recipes-app/example-raw/files/postinst | 13 -------------
2 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 86e2384..b025a7f 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -174,4 +174,14 @@ ISAR_CROSS_COMPILE ?= "0"
# Password was encrypted using following command:
# mkpasswd -m sha512crypt -R 10000
# mkpasswd is part of the 'whois' package of Debian
-CFG_ROOT_PW ?= "$6$rounds=10000$RXeWrnFmkY$DtuS/OmsAS2cCEDo0BF5qQsizIrq6jPgXnwv3PHqREJeKd1sXdHX/ayQtuQWVDHe0KIO0/sVH8dvQm1KthF0d/"
+USERS += "root"
+USER_root[password] ??= "$6$rounds=10000$RXeWrnFmkY$DtuS/OmsAS2cCEDo0BF5qQsizIrq6jPgXnwv3PHqREJeKd1sXdHX/ayQtuQWVDHe0KIO0/sVH8dvQm1KthF0d/"
+
+GROUPS += "isar"
+GROUP_isar[flags] = "system"
+
+USERS += "isar"
+USER_isar[gid] = "isar"
+USER_isar[home] = "/var/lib/isar"
+USER_isar[comment] = "My isar user"
+USER_isar[flags] = "system create-home"
diff --git a/meta-isar/recipes-app/example-raw/files/postinst b/meta-isar/recipes-app/example-raw/files/postinst
index f48d993..416ff34 100644
--- a/meta-isar/recipes-app/example-raw/files/postinst
+++ b/meta-isar/recipes-app/example-raw/files/postinst
@@ -2,17 +2,4 @@
set -e
-if ! getent group isar >/dev/null; then
- addgroup --quiet --system isar
-fi
-
-if ! getent passwd isar >/dev/null; then
- useradd --system --gid isar --create-home \
- --home /var/lib/isar --no-user-group \
- --comment "My isar user" \
- isar
-fi
-
-chown -R isar:isar /var/lib/isar
-
echo "isar" > /etc/hostname
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* [PATCH v4 8/8] RECIPE-API-CHANGELOG: update transient package removal + root password
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (6 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 7/8] meta-isar: local.conf.sample: update root password and isar user creation claudius.heine.ext
@ 2019-05-23 14:55 ` claudius.heine.ext
2019-05-24 12:56 ` [PATCH v4 0/8] pre-processing pipeline and transient package replacement Maxim Yu. Osipov
2019-05-27 15:22 ` Maxim Yu. Osipov
9 siblings, 0 replies; 30+ messages in thread
From: claudius.heine.ext @ 2019-05-23 14:55 UTC (permalink / raw)
To: isar-users; +Cc: Claudius Heine
From: Claudius Heine <ch@denx.de>
Signed-off-by: Claudius Heine <ch@denx.de>
---
RECIPE-API-CHANGELOG.md | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md
index d389d99..bbef1a3 100644
--- a/RECIPE-API-CHANGELOG.md
+++ b/RECIPE-API-CHANGELOG.md
@@ -154,3 +154,25 @@ Changes in v0.8
The content of `isar-image.bbclass` was moved to the `image.bbclass` file.
Recipes that inherit `isar-image` should be modified to inherit from `image`
instead.
+
+### Transient package support was removed
+
+The `LOCALE_GEN` and `LOCALE_DEFAULT` variables are now handled by the
+`image-locales-extension` class within the image recipe.
+
+Setting of the root password can now be done by the `image-account-extension`
+class within the image recipe. To set the root password to empty, you can
+use this code snippet:
+
+```
+USERS += "root"
+USER_root[password] = ""
+USER_root[flags] = "allow-empty-password"
+```
+
+Otherwise set a encrypted root password like this:
+
+```
+USERS += "root"
+USER_root[password] = "$6$rounds=10000$RXeWrnFmkY$DtuS/OmsAS2cCEDo0BF5qQsizIrq6jPgXnwv3PHqREJeKd1sXdHX/ayQtuQWVDHe0KIO0/sVH8dvQm1KthF0d/"
+```
--
2.20.1
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (7 preceding siblings ...)
2019-05-23 14:55 ` [PATCH v4 8/8] RECIPE-API-CHANGELOG: update transient package removal + root password claudius.heine.ext
@ 2019-05-24 12:56 ` Maxim Yu. Osipov
2019-05-27 7:56 ` Claudius Heine
2019-05-27 15:22 ` Maxim Yu. Osipov
9 siblings, 1 reply; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-24 12:56 UTC (permalink / raw)
To: claudius.heine.ext, isar-users; +Cc: Claudius Heine
Hi Claudius,
CI passed OK.
But when I tried to manually test SDK creation procedure (as exactly
described in user_manual) it failed:
I've mounted needed directories with
'mount_chroot.sh sdk-debian-stretch-armhf
and chroot'ed into sdk:
root@tiberius:/# apt-get install libhello-dev:armhf
Reading package lists... Done
Building dependency tree
Reading state information... Done
The following additional packages will be installed:
gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
Suggested packages:
glibc-doc:armhf locales:armhf
The following NEW packages will be installed:
gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
libhello-dev:armhf
0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
Need to get 0 B/2,564 kB of archives.
After this operation, 8,987 kB of additional disk space will be used.
Do you want to continue? [Y/n]
Get:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
6.3.0-18+deb9u1 [180 kB]
Err:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
6.3.0-18+deb9u1
File not found -
/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
(2: No such file or directory)
Get:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
1:6.3.0-18+deb9u1 [38.5 kB]
Err:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
1:6.3.0-18+deb9u1
File not found -
/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb (2:
No such file or directory)
Get:3 file:/base-apt/debian stretch/main armhf libc6 armhf
2.24-11+deb9u4 [2,341 kB]
Err:3 file:/base-apt/debian stretch/main armhf libc6 armhf 2.24-11+deb9u4
File not found -
/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2: No
such file or directory)
Get:4 file:/isar-apt isar/main armhf libhello armhf 0.1 [2,826 B]
Get:5 file:/isar-apt isar/main armhf libhello-dev armhf 0.1 [2,418 B]
E: Failed to fetch
file:/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
File not found -
/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
(2: No such file or directory)
E: Failed to fetch
file:/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
File not found -
/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb (2:
No such file or directory)
E: Failed to fetch
file:/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
File not found -
/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2: No
such file or directory)
E: Unable to fetch some archives, maybe run apt-get update or try with
--fix-missing?
root@tiberius:/# apt-get update
Get:1 file:/base-apt/debian stretch InRelease
Ign:1 file:/base-apt/debian stretch InRelease
Get:2 file:/isar-apt isar InRelease
Ign:2 file:/isar-apt isar InRelease
Get:3 file:/base-apt/debian stretch Release
Ign:3 file:/base-apt/debian stretch Release
Get:4 file:/isar-apt isar Release [3,584 B]
Get:5 file:/base-apt/debian stretch/main armhf Packages
Ign:5 file:/base-apt/debian stretch/main armhf Packages
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Get:7 file:/base-apt/debian stretch/main amd64 Packages
Ign:7 file:/base-apt/debian stretch/main amd64 Packages
Get:8 file:/base-apt/debian stretch/main Translation-en_US
Ign:8 file:/base-apt/debian stretch/main Translation-en_US
Get:9 file:/base-apt/debian stretch/main Translation-en
Ign:9 file:/base-apt/debian stretch/main Translation-en
Get:4 file:/isar-apt isar Release [3,584 B]
Get:5 file:/base-apt/debian stretch/main armhf Packages
Ign:5 file:/base-apt/debian stretch/main armhf Packages
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Get:7 file:/base-apt/debian stretch/main amd64 Packages
Ign:7 file:/base-apt/debian stretch/main amd64 Packages
Get:10 file:/isar-apt isar Release.gpg
Ign:10 file:/isar-apt isar Release.gpg
Get:8 file:/base-apt/debian stretch/main Translation-en_US
Ign:8 file:/base-apt/debian stretch/main Translation-en_US
Get:9 file:/base-apt/debian stretch/main Translation-en
Ign:9 file:/base-apt/debian stretch/main Translation-en
Get:5 file:/base-apt/debian stretch/main armhf Packages
Ign:5 file:/base-apt/debian stretch/main armhf Packages
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Get:7 file:/base-apt/debian stretch/main amd64 Packages
Ign:7 file:/base-apt/debian stretch/main amd64 Packages
Get:8 file:/base-apt/debian stretch/main Translation-en_US
Ign:8 file:/base-apt/debian stretch/main Translation-en_US
Get:9 file:/base-apt/debian stretch/main Translation-en
Ign:9 file:/base-apt/debian stretch/main Translation-en
Get:5 file:/base-apt/debian stretch/main armhf Packages
Ign:5 file:/base-apt/debian stretch/main armhf Packages
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Get:7 file:/base-apt/debian stretch/main amd64 Packages
Ign:7 file:/base-apt/debian stretch/main amd64 Packages
Get:8 file:/base-apt/debian stretch/main Translation-en_US
Ign:8 file:/base-apt/debian stretch/main Translation-en_US
Get:9 file:/base-apt/debian stretch/main Translation-en
Ign:9 file:/base-apt/debian stretch/main Translation-en
Get:5 file:/base-apt/debian stretch/main armhf Packages
Ign:5 file:/base-apt/debian stretch/main armhf Packages
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Get:7 file:/base-apt/debian stretch/main amd64 Packages
Ign:7 file:/base-apt/debian stretch/main amd64 Packages
Get:8 file:/base-apt/debian stretch/main Translation-en_US
Ign:8 file:/base-apt/debian stretch/main Translation-en_US
Get:9 file:/base-apt/debian stretch/main Translation-en
Ign:9 file:/base-apt/debian stretch/main Translation-en
Get:5 file:/base-apt/debian stretch/main armhf Packages
Err:5 file:/base-apt/debian stretch/main armhf Packages
File not found -
/base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No such
file or directory)
Get:6 file:/base-apt/debian stretch/main all Packages
Ign:6 file:/base-apt/debian stretch/main all Packages
Reading package lists... Done
E: Failed to fetch
file:/base-apt/debian/dists/stretch/main/binary-armhf/Packages File not
found - /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No
such file or directory)
E: Some index files failed to download. They have been ignored, or old
ones used instead.
Regards,
Maxim.
On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> Hi,
>
> I decided to heed Hennings advice and order the removal patch before the split
> patch. This way the transient package removal patch becomes more complete.
>
> regards,
> Claudius
>
> changes from v3:
> - ordered the transient package removal patch before the split
>
> changes from v2:
> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>
> changes from v1:
> - rebased to current next
> - added some more documentation
>
> Claudius Heine (8):
> meta: remove transient package support
> split up isar-bootstrap helper and implement pre-process pipeline
> meta/classes: add image-locales-extension class
> meta/classes: add image-account-extension class
> doc: update description of image customization
> doc: some fixes
> meta-isar: local.conf.sample: update root password and isar user
> creation
> RECIPE-API-CHANGELOG: update transient package removal + root password
>
> RECIPE-API-CHANGELOG.md | 22 ++
> doc/user_manual.md | 53 +++-
> meta-isar/conf/local.conf.sample | 12 +-
> .../recipes-app/example-raw/files/postinst | 13 -
> meta/classes/image-account-extension.bbclass | 257 ++++++++++++++++++
> meta/classes/image-locales-extension.bbclass | 97 +++++++
> meta/classes/image.bbclass | 30 +-
> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
> meta/classes/rootfs.bbclass | 176 +++++++++++-
> .../isar-bootstrap/isar-bootstrap.inc | 5 +
> .../buildchroot/buildchroot-host.bb | 16 +-
> .../buildchroot/buildchroot-target.bb | 2 -
> .../buildchroot/buildchroot.inc | 26 +-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
> .../isar-cfg-localepurge/files/postinst | 15 -
> .../isar-cfg-localepurge.bb | 70 -----
> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
> 18 files changed, 663 insertions(+), 360 deletions(-)
> create mode 100644 meta/classes/image-account-extension.bbclass
> create mode 100644 meta/classes/image-locales-extension.bbclass
> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
> delete mode 100644 meta/recipes-support/isar-cfg-localepurge/files/postinst
> delete mode 100644 meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
> delete mode 100644 meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
> delete mode 100644 meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-24 12:56 ` [PATCH v4 0/8] pre-processing pipeline and transient package replacement Maxim Yu. Osipov
@ 2019-05-27 7:56 ` Claudius Heine
2019-05-27 8:10 ` Maxim Yu. Osipov
0 siblings, 1 reply; 30+ messages in thread
From: Claudius Heine @ 2019-05-27 7:56 UTC (permalink / raw)
To: Maxim Yu. Osipov, isar-users; +Cc: Claudius Heine
Hi Maxim,
On 24/05/2019 14.56, Maxim Yu. Osipov wrote:
> Hi Claudius,
>
> CI passed OK.
>
> But when I tried to manually test SDK creation procedure (as exactly
> described in user_manual) it failed:
Have you tested if it does not fail the same way in next when using the
base-apt repository?
regards,
Claudius
>
> I've mounted needed directories with
> 'mount_chroot.sh sdk-debian-stretch-armhf
>
> and chroot'ed into sdk:
>
> root@tiberius:/# apt-get install libhello-dev:armhf
> Reading package lists... Done
> Building dependency tree
> Reading state information... Done
> The following additional packages will be installed:
> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
> Suggested packages:
> glibc-doc:armhf locales:armhf
> The following NEW packages will be installed:
> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
> libhello-dev:armhf
> 0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
> Need to get 0 B/2,564 kB of archives.
> After this operation, 8,987 kB of additional disk space will be used.
> Do you want to continue? [Y/n]
> Get:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
> 6.3.0-18+deb9u1 [180 kB]
> Err:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
> 6.3.0-18+deb9u1
> File not found -
> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
> (2: No such file or directory)
> Get:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
> 1:6.3.0-18+deb9u1 [38.5 kB]
> Err:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
> 1:6.3.0-18+deb9u1
> File not found -
> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb (2:
> No such file or directory)
> Get:3 file:/base-apt/debian stretch/main armhf libc6 armhf
> 2.24-11+deb9u4 [2,341 kB]
> Err:3 file:/base-apt/debian stretch/main armhf libc6 armhf 2.24-11+deb9u4
> File not found -
> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2: No
> such file or directory)
> Get:4 file:/isar-apt isar/main armhf libhello armhf 0.1 [2,826 B]
> Get:5 file:/isar-apt isar/main armhf libhello-dev armhf 0.1 [2,418 B]
> E: Failed to fetch
> file:/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
> File not found -
> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
> (2: No such file or directory)
> E: Failed to fetch
> file:/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb File
> not found -
> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb (2:
> No such file or directory)
> E: Failed to fetch
> file:/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
> File not found -
> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2: No
> such file or directory)
> E: Unable to fetch some archives, maybe run apt-get update or try with
> --fix-missing?
> root@tiberius:/# apt-get update
> Get:1 file:/base-apt/debian stretch InRelease
> Ign:1 file:/base-apt/debian stretch InRelease
> Get:2 file:/isar-apt isar InRelease
> Ign:2 file:/isar-apt isar InRelease
> Get:3 file:/base-apt/debian stretch Release
> Ign:3 file:/base-apt/debian stretch Release
> Get:4 file:/isar-apt isar Release [3,584 B]
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Ign:5 file:/base-apt/debian stretch/main armhf Packages
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Get:7 file:/base-apt/debian stretch/main amd64 Packages
> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
> Get:8 file:/base-apt/debian stretch/main Translation-en_US
> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
> Get:9 file:/base-apt/debian stretch/main Translation-en
> Ign:9 file:/base-apt/debian stretch/main Translation-en
> Get:4 file:/isar-apt isar Release [3,584 B]
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Ign:5 file:/base-apt/debian stretch/main armhf Packages
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Get:7 file:/base-apt/debian stretch/main amd64 Packages
> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
> Get:10 file:/isar-apt isar Release.gpg
> Ign:10 file:/isar-apt isar Release.gpg
> Get:8 file:/base-apt/debian stretch/main Translation-en_US
> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
> Get:9 file:/base-apt/debian stretch/main Translation-en
> Ign:9 file:/base-apt/debian stretch/main Translation-en
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Ign:5 file:/base-apt/debian stretch/main armhf Packages
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Get:7 file:/base-apt/debian stretch/main amd64 Packages
> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
> Get:8 file:/base-apt/debian stretch/main Translation-en_US
> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
> Get:9 file:/base-apt/debian stretch/main Translation-en
> Ign:9 file:/base-apt/debian stretch/main Translation-en
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Ign:5 file:/base-apt/debian stretch/main armhf Packages
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Get:7 file:/base-apt/debian stretch/main amd64 Packages
> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
> Get:8 file:/base-apt/debian stretch/main Translation-en_US
> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
> Get:9 file:/base-apt/debian stretch/main Translation-en
> Ign:9 file:/base-apt/debian stretch/main Translation-en
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Ign:5 file:/base-apt/debian stretch/main armhf Packages
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Get:7 file:/base-apt/debian stretch/main amd64 Packages
> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
> Get:8 file:/base-apt/debian stretch/main Translation-en_US
> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
> Get:9 file:/base-apt/debian stretch/main Translation-en
> Ign:9 file:/base-apt/debian stretch/main Translation-en
> Get:5 file:/base-apt/debian stretch/main armhf Packages
> Err:5 file:/base-apt/debian stretch/main armhf Packages
> File not found -
> /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No such
> file or directory)
> Get:6 file:/base-apt/debian stretch/main all Packages
> Ign:6 file:/base-apt/debian stretch/main all Packages
> Reading package lists... Done
> E: Failed to fetch
> file:/base-apt/debian/dists/stretch/main/binary-armhf/Packages File not
> found - /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No
> such file or directory)
> E: Some index files failed to download. They have been ignored, or old
> ones used instead.
>
>
> Regards,
> Maxim.
>
> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>> From: Claudius Heine <ch@denx.de>
>>
>> Hi,
>>
>> I decided to heed Hennings advice and order the removal patch before
>> the split
>> patch. This way the transient package removal patch becomes more
>> complete.
>>
>> regards,
>> Claudius
>>
>> changes from v3:
>> - ordered the transient package removal patch before the split
>>
>> changes from v2:
>> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>>
>> changes from v1:
>> - rebased to current next
>> - added some more documentation
>>
>> Claudius Heine (8):
>> meta: remove transient package support
>> split up isar-bootstrap helper and implement pre-process pipeline
>> meta/classes: add image-locales-extension class
>> meta/classes: add image-account-extension class
>> doc: update description of image customization
>> doc: some fixes
>> meta-isar: local.conf.sample: update root password and isar user
>> creation
>> RECIPE-API-CHANGELOG: update transient package removal + root password
>>
>> RECIPE-API-CHANGELOG.md | 22 ++
>> doc/user_manual.md | 53 +++-
>> meta-isar/conf/local.conf.sample | 12 +-
>> .../recipes-app/example-raw/files/postinst | 13 -
>> meta/classes/image-account-extension.bbclass | 257 ++++++++++++++++++
>> meta/classes/image-locales-extension.bbclass | 97 +++++++
>> meta/classes/image.bbclass | 30 +-
>> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
>> meta/classes/rootfs.bbclass | 176 +++++++++++-
>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>> .../buildchroot/buildchroot-host.bb | 16 +-
>> .../buildchroot/buildchroot-target.bb | 2 -
>> .../buildchroot/buildchroot.inc | 26 +-
>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
>> .../isar-cfg-localepurge/files/postinst | 15 -
>> .../isar-cfg-localepurge.bb | 70 -----
>> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
>> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
>> 18 files changed, 663 insertions(+), 360 deletions(-)
>> create mode 100644 meta/classes/image-account-extension.bbclass
>> create mode 100644 meta/classes/image-locales-extension.bbclass
>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>> delete mode 100644
>> meta/recipes-support/isar-cfg-localepurge/files/postinst
>> delete mode 100644
>> meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
>> delete mode 100644
>> meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
>> delete mode 100644
>> meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>>
>
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-27 7:56 ` Claudius Heine
@ 2019-05-27 8:10 ` Maxim Yu. Osipov
2019-05-27 8:24 ` Claudius Heine
0 siblings, 1 reply; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 8:10 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
Hi Claudius,
On 5/27/19 9:56 AM, Claudius Heine wrote:
> Hi Maxim,
>
> On 24/05/2019 14.56, Maxim Yu. Osipov wrote:
>> Hi Claudius,
>>
>> CI passed OK.
>>
>> But when I tried to manually test SDK creation procedure (as exactly
>> described in user_manual) it failed:
>
> Have you tested if it does not fail the same way in next when using the
> base-apt repository?
I've performed the following steps (actually exactly followed current
SDK creation description in user_manual):
1) In clean tree with your patches applied, I've enabled cross
compilation in local.conf and launched SDK creation procedure for
qemuarm-stretch
'build bitbake -c do_populate_sdk
multiconfig:qemuarm-stretch:isar-image-base'
The build was OK (as expected because such SDK creation build passed in CI).
2) I've mounted needed directories mount_chroot.sh
sdk-debian-stretch-armhf and chrooted into in.
3) In SDK chroot commands apt-get update and apt-get install
libhello-dev:armhf failed (logs are below).
I've performed the same procedure w/o your patches - it works as
described in user_manual.
Regards,
Maxim.
> regards,
> Claudius
>
>>
>> I've mounted needed directories with
>> 'mount_chroot.sh sdk-debian-stretch-armhf
>>
>> and chroot'ed into sdk:
>>
>> root@tiberius:/# apt-get install libhello-dev:armhf
>> Reading package lists... Done
>> Building dependency tree
>> Reading state information... Done
>> The following additional packages will be installed:
>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>> Suggested packages:
>> glibc-doc:armhf locales:armhf
>> The following NEW packages will be installed:
>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>> libhello-dev:armhf
>> 0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
>> Need to get 0 B/2,564 kB of archives.
>> After this operation, 8,987 kB of additional disk space will be used.
>> Do you want to continue? [Y/n]
>> Get:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>> 6.3.0-18+deb9u1 [180 kB]
>> Err:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>> 6.3.0-18+deb9u1
>> File not found -
>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb (2:
>> No such file or directory)
>> Get:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>> 1:6.3.0-18+deb9u1 [38.5 kB]
>> Err:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>> 1:6.3.0-18+deb9u1
>> File not found -
>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>> (2: No such file or directory)
>> Get:3 file:/base-apt/debian stretch/main armhf libc6 armhf
>> 2.24-11+deb9u4 [2,341 kB]
>> Err:3 file:/base-apt/debian stretch/main armhf libc6 armhf 2.24-11+deb9u4
>> File not found -
>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2:
>> No such file or directory)
>> Get:4 file:/isar-apt isar/main armhf libhello armhf 0.1 [2,826 B]
>> Get:5 file:/isar-apt isar/main armhf libhello-dev armhf 0.1 [2,418 B]
>> E: Failed to fetch
>> file:/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>> File not found -
>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb (2:
>> No such file or directory)
>> E: Failed to fetch
>> file:/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>> File not found -
>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>> (2: No such file or directory)
>> E: Failed to fetch
>> file:/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
>> File not found -
>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2:
>> No such file or directory)
>> E: Unable to fetch some archives, maybe run apt-get update or try with
>> --fix-missing?
>> root@tiberius:/# apt-get update
>> Get:1 file:/base-apt/debian stretch InRelease
>> Ign:1 file:/base-apt/debian stretch InRelease
>> Get:2 file:/isar-apt isar InRelease
>> Ign:2 file:/isar-apt isar InRelease
>> Get:3 file:/base-apt/debian stretch Release
>> Ign:3 file:/base-apt/debian stretch Release
>> Get:4 file:/isar-apt isar Release [3,584 B]
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>> Get:9 file:/base-apt/debian stretch/main Translation-en
>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>> Get:4 file:/isar-apt isar Release [3,584 B]
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>> Get:10 file:/isar-apt isar Release.gpg
>> Ign:10 file:/isar-apt isar Release.gpg
>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>> Get:9 file:/base-apt/debian stretch/main Translation-en
>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>> Get:9 file:/base-apt/debian stretch/main Translation-en
>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>> Get:9 file:/base-apt/debian stretch/main Translation-en
>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>> Get:9 file:/base-apt/debian stretch/main Translation-en
>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>> Err:5 file:/base-apt/debian stretch/main armhf Packages
>> File not found -
>> /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No such
>> file or directory)
>> Get:6 file:/base-apt/debian stretch/main all Packages
>> Ign:6 file:/base-apt/debian stretch/main all Packages
>> Reading package lists... Done
>> E: Failed to fetch
>> file:/base-apt/debian/dists/stretch/main/binary-armhf/Packages File
>> not found - /base-apt/debian/dists/stretch/main/binary-armhf/Packages
>> (2: No such file or directory)
>> E: Some index files failed to download. They have been ignored, or old
>> ones used instead.
>>
>>
>> Regards,
>> Maxim.
>>
>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>> From: Claudius Heine <ch@denx.de>
>>>
>>> Hi,
>>>
>>> I decided to heed Hennings advice and order the removal patch before
>>> the split
>>> patch. This way the transient package removal patch becomes more
>>> complete.
>>>
>>> regards,
>>> Claudius
>>>
>>> changes from v3:
>>> - ordered the transient package removal patch before the split
>>>
>>> changes from v2:
>>> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>>>
>>> changes from v1:
>>> - rebased to current next
>>> - added some more documentation
>>>
>>> Claudius Heine (8):
>>> meta: remove transient package support
>>> split up isar-bootstrap helper and implement pre-process pipeline
>>> meta/classes: add image-locales-extension class
>>> meta/classes: add image-account-extension class
>>> doc: update description of image customization
>>> doc: some fixes
>>> meta-isar: local.conf.sample: update root password and isar user
>>> creation
>>> RECIPE-API-CHANGELOG: update transient package removal + root
>>> password
>>>
>>> RECIPE-API-CHANGELOG.md | 22 ++
>>> doc/user_manual.md | 53 +++-
>>> meta-isar/conf/local.conf.sample | 12 +-
>>> .../recipes-app/example-raw/files/postinst | 13 -
>>> meta/classes/image-account-extension.bbclass | 257 ++++++++++++++++++
>>> meta/classes/image-locales-extension.bbclass | 97 +++++++
>>> meta/classes/image.bbclass | 30 +-
>>> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
>>> meta/classes/rootfs.bbclass | 176 +++++++++++-
>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>> .../buildchroot/buildchroot-target.bb | 2 -
>>> .../buildchroot/buildchroot.inc | 26 +-
>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
>>> .../isar-cfg-localepurge/files/postinst | 15 -
>>> .../isar-cfg-localepurge.bb | 70 -----
>>> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
>>> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
>>> 18 files changed, 663 insertions(+), 360 deletions(-)
>>> create mode 100644 meta/classes/image-account-extension.bbclass
>>> create mode 100644 meta/classes/image-locales-extension.bbclass
>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>> delete mode 100644
>>> meta/recipes-support/isar-cfg-localepurge/files/postinst
>>> delete mode 100644
>>> meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
>>> delete mode 100644
>>> meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
>>> delete mode 100644
>>> meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>>>
>>
>>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-27 8:10 ` Maxim Yu. Osipov
@ 2019-05-27 8:24 ` Claudius Heine
2019-05-27 9:28 ` Maxim Yu. Osipov
0 siblings, 1 reply; 30+ messages in thread
From: Claudius Heine @ 2019-05-27 8:24 UTC (permalink / raw)
To: Maxim Yu. Osipov, isar-users; +Cc: Claudius Heine
Hi Maxim,
On 27/05/2019 10.10, Maxim Yu. Osipov wrote:
> Hi Claudius,
>
> On 5/27/19 9:56 AM, Claudius Heine wrote:
>> Hi Maxim,
>>
>> On 24/05/2019 14.56, Maxim Yu. Osipov wrote:
>>> Hi Claudius,
>>>
>>> CI passed OK.
>>>
>>> But when I tried to manually test SDK creation procedure (as exactly
>>> described in user_manual) it failed:
>>
>> Have you tested if it does not fail the same way in next when using
>> the base-apt repository?
>
> I've performed the following steps (actually exactly followed current
> SDK creation description in user_manual):
>
> 1) In clean tree with your patches applied, I've enabled cross
> compilation in local.conf and launched SDK creation procedure for
> qemuarm-stretch
> 'build bitbake -c do_populate_sdk
> multiconfig:qemuarm-stretch:isar-image-base'
>
> The build was OK (as expected because such SDK creation build passed in
> CI).
>
> 2) I've mounted needed directories mount_chroot.sh
> sdk-debian-stretch-armhf and chrooted into in.
>
> 3) In SDK chroot commands apt-get update and apt-get install
> libhello-dev:armhf failed (logs are below).
>
>
> I've performed the same procedure w/o your patches - it works as
> described in user_manual.
Well that does not answer my question wherever or not you have enabled
ISAR_USE_CACHED_BASE_REPO, in both builds. The logs mention
'file:/base-apt/debian' so I am reasonable sure that you had
`ISAR_USE_CACHED_BASE_REPO ="1"` and I just tried that with a filled
base-apt in current next resulting in the same error.
>
> Regards,
> Maxim.
>
>
>> regards,
>> Claudius
>>
>>>
>>> I've mounted needed directories with
>>> 'mount_chroot.sh sdk-debian-stretch-armhf
>>>
>>> and chroot'ed into sdk:
>>>
>>> root@tiberius:/# apt-get install libhello-dev:armhf
>>> Reading package lists... Done
>>> Building dependency tree
>>> Reading state information... Done
>>> The following additional packages will be installed:
>>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>>> Suggested packages:
>>> glibc-doc:armhf locales:armhf
>>> The following NEW packages will be installed:
>>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>>> libhello-dev:armhf
>>> 0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
>>> Need to get 0 B/2,564 kB of archives.
>>> After this operation, 8,987 kB of additional disk space will be used.
>>> Do you want to continue? [Y/n]
>>> Get:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>>> 6.3.0-18+deb9u1 [180 kB]
>>> Err:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>>> 6.3.0-18+deb9u1
>>> File not found -
>>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>> (2: No such file or directory)
>>> Get:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>>> 1:6.3.0-18+deb9u1 [38.5 kB]
>>> Err:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>>> 1:6.3.0-18+deb9u1
>>> File not found -
>>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>> (2: No such file or directory)
>>> Get:3 file:/base-apt/debian stretch/main armhf libc6 armhf
>>> 2.24-11+deb9u4 [2,341 kB]
>>> Err:3 file:/base-apt/debian stretch/main armhf libc6 armhf
>>> 2.24-11+deb9u4
>>> File not found -
>>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2:
>>> No such file or directory)
>>> Get:4 file:/isar-apt isar/main armhf libhello armhf 0.1 [2,826 B]
>>> Get:5 file:/isar-apt isar/main armhf libhello-dev armhf 0.1 [2,418 B]
>>> E: Failed to fetch
>>> file:/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>> File not found -
>>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>> (2: No such file or directory)
>>> E: Failed to fetch
>>> file:/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>> File not found -
>>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>> (2: No such file or directory)
>>> E: Failed to fetch
>>> file:/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb File
>>> not found -
>>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb (2:
>>> No such file or directory)
>>> E: Unable to fetch some archives, maybe run apt-get update or try
>>> with --fix-missing?
>>> root@tiberius:/# apt-get update
>>> Get:1 file:/base-apt/debian stretch InRelease
>>> Ign:1 file:/base-apt/debian stretch InRelease
>>> Get:2 file:/isar-apt isar InRelease
>>> Ign:2 file:/isar-apt isar InRelease
>>> Get:3 file:/base-apt/debian stretch Release
>>> Ign:3 file:/base-apt/debian stretch Release
>>> Get:4 file:/isar-apt isar Release [3,584 B]
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>> Get:4 file:/isar-apt isar Release [3,584 B]
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Get:10 file:/isar-apt isar Release.gpg
>>> Ign:10 file:/isar-apt isar Release.gpg
>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>> Err:5 file:/base-apt/debian stretch/main armhf Packages
>>> File not found -
>>> /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No such
>>> file or directory)
>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>> Reading package lists... Done
>>> E: Failed to fetch
>>> file:/base-apt/debian/dists/stretch/main/binary-armhf/Packages File
>>> not found - /base-apt/debian/dists/stretch/main/binary-armhf/Packages
>>> (2: No such file or directory)
>>> E: Some index files failed to download. They have been ignored, or
>>> old ones used instead.
>>>
>>>
>>> Regards,
>>> Maxim.
>>>
>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>> From: Claudius Heine <ch@denx.de>
>>>>
>>>> Hi,
>>>>
>>>> I decided to heed Hennings advice and order the removal patch before
>>>> the split
>>>> patch. This way the transient package removal patch becomes more
>>>> complete.
>>>>
>>>> regards,
>>>> Claudius
>>>>
>>>> changes from v3:
>>>> - ordered the transient package removal patch before the split
>>>>
>>>> changes from v2:
>>>> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>>>>
>>>> changes from v1:
>>>> - rebased to current next
>>>> - added some more documentation
>>>>
>>>> Claudius Heine (8):
>>>> meta: remove transient package support
>>>> split up isar-bootstrap helper and implement pre-process pipeline
>>>> meta/classes: add image-locales-extension class
>>>> meta/classes: add image-account-extension class
>>>> doc: update description of image customization
>>>> doc: some fixes
>>>> meta-isar: local.conf.sample: update root password and isar user
>>>> creation
>>>> RECIPE-API-CHANGELOG: update transient package removal + root
>>>> password
>>>>
>>>> RECIPE-API-CHANGELOG.md | 22 ++
>>>> doc/user_manual.md | 53 +++-
>>>> meta-isar/conf/local.conf.sample | 12 +-
>>>> .../recipes-app/example-raw/files/postinst | 13 -
>>>> meta/classes/image-account-extension.bbclass | 257
>>>> ++++++++++++++++++
>>>> meta/classes/image-locales-extension.bbclass | 97 +++++++
>>>> meta/classes/image.bbclass | 30 +-
>>>> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
>>>> meta/classes/rootfs.bbclass | 176 +++++++++++-
>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>> .../buildchroot/buildchroot.inc | 26 +-
>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
>>>> .../isar-cfg-localepurge/files/postinst | 15 -
>>>> .../isar-cfg-localepurge.bb | 70 -----
>>>> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
>>>> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
>>>> 18 files changed, 663 insertions(+), 360 deletions(-)
>>>> create mode 100644 meta/classes/image-account-extension.bbclass
>>>> create mode 100644 meta/classes/image-locales-extension.bbclass
>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>> delete mode 100644
>>>> meta/recipes-support/isar-cfg-localepurge/files/postinst
>>>> delete mode 100644
>>>> meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
>>>> delete mode 100644
>>>> meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
>>>> delete mode 100644
>>>> meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>>>>
>>>
>>>
>>
>
>
--
DENX Software Engineering GmbH, Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-27 8:24 ` Claudius Heine
@ 2019-05-27 9:28 ` Maxim Yu. Osipov
0 siblings, 0 replies; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 9:28 UTC (permalink / raw)
To: Claudius Heine, isar-users; +Cc: Claudius Heine
[-- Attachment #1: Type: text/plain, Size: 14310 bytes --]
Hi Claudius,
On 5/27/19 10:24 AM, Claudius Heine wrote:
> Hi Maxim,
>
> On 27/05/2019 10.10, Maxim Yu. Osipov wrote:
>> Hi Claudius,
>>
>> On 5/27/19 9:56 AM, Claudius Heine wrote:
>>> Hi Maxim,
>>>
>>> On 24/05/2019 14.56, Maxim Yu. Osipov wrote:
>>>> Hi Claudius,
>>>>
>>>> CI passed OK.
>>>>
>>>> But when I tried to manually test SDK creation procedure (as exactly
>>>> described in user_manual) it failed:
>>>
>>> Have you tested if it does not fail the same way in next when using
>>> the base-apt repository?
>>
>> I've performed the following steps (actually exactly followed current
>> SDK creation description in user_manual):
>>
>> 1) In clean tree with your patches applied, I've enabled cross
>> compilation in local.conf and launched SDK creation procedure for
>> qemuarm-stretch
>> 'build bitbake -c do_populate_sdk
>> multiconfig:qemuarm-stretch:isar-image-base'
>>
>> The build was OK (as expected because such SDK creation build passed
>> in CI).
>>
>> 2) I've mounted needed directories mount_chroot.sh
>> sdk-debian-stretch-armhf and chrooted into in.
>>
>> 3) In SDK chroot commands apt-get update and apt-get install
>> libhello-dev:armhf failed (logs are below).
>>
>>
>> I've performed the same procedure w/o your patches - it works as
>> described in user_manual.
>
> Well that does not answer my question wherever or not you have enabled
> ISAR_USE_CACHED_BASE_REPO, in both builds. The logs mention
> 'file:/base-apt/debian' so I am reasonable sure that you had
> `ISAR_USE_CACHED_BASE_REPO ="1"` and I just tried that with a filled
> base-apt in current next resulting in the same error.
You are right - I've just double checked from the scratch (my current
local.conf is attached) - the only thing now is that resolv.conf is
missing to sdkchroot, so 'apt-get update' complains (see below). When I
copied resolv.conf from my host to sdk chroot 'apt-get update/apt-get
install libhello-dev:armhf' in chroot succeeded.
It's not a bug as sdk chroot may be distributed to other machines (or PC
can be connected later to other network) and resolv.conf has to be
substituted accordingly.
I'll mention this explicitly in user manual.
root@tiberius:/# dpkg -l | grep essen
ii crossbuild-essential-armhf 12.3 all
Informational list of cross-build-essential packages
root@tiberius:/# apt-get update
Get:1 file:/isar-apt isar InRelease
Ign:1 file:/isar-apt isar InRelease
Get:2 file:/isar-apt isar Release [3,569 B]
Err:3 http://ftp.debian.org/debian stretch InRelease
Temporary failure resolving 'ftp.debian.org'
Err:4 http://ftp.debian.org/debian stretch-updates InRelease
Temporary failure resolving 'ftp.debian.org'
Err:5 http://security.debian.org stretch/updates InRelease
Temporary failure resolving 'security.debian.org'
Get:2 file:/isar-apt isar Release [3,569 B]
Get:6 file:/isar-apt isar Release.gpg
Ign:6 file:/isar-apt isar Release.gpg
Get:7 file:/isar-apt isar/main armhf Packages [1,666 B]
Reading package lists... Done
W: Failed to fetch http://ftp.debian.org/debian/dists/stretch/InRelease
Temporary failure resolving 'ftp.debian.org'
W: Failed to fetch
http://ftp.debian.org/debian/dists/stretch-updates/InRelease Temporary
failure resolving 'ftp.debian.org'
W: Failed to fetch
http://security.debian.org/dists/stretch/updates/InRelease Temporary
failure resolving 'security.debian.org'
W: Some index files failed to download. They have been ignored, or old
ones used instead.
ls -l /etc/resolv.conf
ls: cannot access '/etc/resolv.conf': No such file or directory
root@tiberius:/#
Regards,
Maxim.
>>
>> Regards,
>> Maxim.
>>
>>
>>> regards,
>>> Claudius
>>>
>>>>
>>>> I've mounted needed directories with
>>>> 'mount_chroot.sh sdk-debian-stretch-armhf
>>>>
>>>> and chroot'ed into sdk:
>>>>
>>>> root@tiberius:/# apt-get install libhello-dev:armhf
>>>> Reading package lists... Done
>>>> Building dependency tree
>>>> Reading state information... Done
>>>> The following additional packages will be installed:
>>>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>>>> Suggested packages:
>>>> glibc-doc:armhf locales:armhf
>>>> The following NEW packages will be installed:
>>>> gcc-6-base:armhf libc6:armhf libgcc1:armhf libhello:armhf
>>>> libhello-dev:armhf
>>>> 0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
>>>> Need to get 0 B/2,564 kB of archives.
>>>> After this operation, 8,987 kB of additional disk space will be used.
>>>> Do you want to continue? [Y/n]
>>>> Get:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>>>> 6.3.0-18+deb9u1 [180 kB]
>>>> Err:1 file:/base-apt/debian stretch/main armhf gcc-6-base armhf
>>>> 6.3.0-18+deb9u1
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>>> (2: No such file or directory)
>>>> Get:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>>>> 1:6.3.0-18+deb9u1 [38.5 kB]
>>>> Err:2 file:/base-apt/debian stretch/main armhf libgcc1 armhf
>>>> 1:6.3.0-18+deb9u1
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>>> (2: No such file or directory)
>>>> Get:3 file:/base-apt/debian stretch/main armhf libc6 armhf
>>>> 2.24-11+deb9u4 [2,341 kB]
>>>> Err:3 file:/base-apt/debian stretch/main armhf libc6 armhf
>>>> 2.24-11+deb9u4
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
>>>> (2: No such file or directory)
>>>> Get:4 file:/isar-apt isar/main armhf libhello armhf 0.1 [2,826 B]
>>>> Get:5 file:/isar-apt isar/main armhf libhello-dev armhf 0.1 [2,418 B]
>>>> E: Failed to fetch
>>>> file:/base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/gcc-6/gcc-6-base_6.3.0-18+deb9u1_armhf.deb
>>>> (2: No such file or directory)
>>>> E: Failed to fetch
>>>> file:/base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/gcc-6/libgcc1_6.3.0-18+deb9u1_armhf.deb
>>>> (2: No such file or directory)
>>>> E: Failed to fetch
>>>> file:/base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
>>>> File not found -
>>>> /base-apt/debian/pool/main/g/glibc/libc6_2.24-11+deb9u4_armhf.deb
>>>> (2: No such file or directory)
>>>> E: Unable to fetch some archives, maybe run apt-get update or try
>>>> with --fix-missing?
>>>> root@tiberius:/# apt-get update
>>>> Get:1 file:/base-apt/debian stretch InRelease
>>>> Ign:1 file:/base-apt/debian stretch InRelease
>>>> Get:2 file:/isar-apt isar InRelease
>>>> Ign:2 file:/isar-apt isar InRelease
>>>> Get:3 file:/base-apt/debian stretch Release
>>>> Ign:3 file:/base-apt/debian stretch Release
>>>> Get:4 file:/isar-apt isar Release [3,584 B]
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>>> Get:4 file:/isar-apt isar Release [3,584 B]
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Get:10 file:/isar-apt isar Release.gpg
>>>> Ign:10 file:/isar-apt isar Release.gpg
>>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Ign:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Get:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Ign:7 file:/base-apt/debian stretch/main amd64 Packages
>>>> Get:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Ign:8 file:/base-apt/debian stretch/main Translation-en_US
>>>> Get:9 file:/base-apt/debian stretch/main Translation-en
>>>> Ign:9 file:/base-apt/debian stretch/main Translation-en
>>>> Get:5 file:/base-apt/debian stretch/main armhf Packages
>>>> Err:5 file:/base-apt/debian stretch/main armhf Packages
>>>> File not found -
>>>> /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No
>>>> such file or directory)
>>>> Get:6 file:/base-apt/debian stretch/main all Packages
>>>> Ign:6 file:/base-apt/debian stretch/main all Packages
>>>> Reading package lists... Done
>>>> E: Failed to fetch
>>>> file:/base-apt/debian/dists/stretch/main/binary-armhf/Packages File
>>>> not found -
>>>> /base-apt/debian/dists/stretch/main/binary-armhf/Packages (2: No
>>>> such file or directory)
>>>> E: Some index files failed to download. They have been ignored, or
>>>> old ones used instead.
>>>>
>>>>
>>>> Regards,
>>>> Maxim.
>>>>
>>>> On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
>>>>> From: Claudius Heine <ch@denx.de>
>>>>>
>>>>> Hi,
>>>>>
>>>>> I decided to heed Hennings advice and order the removal patch
>>>>> before the split
>>>>> patch. This way the transient package removal patch becomes more
>>>>> complete.
>>>>>
>>>>> regards,
>>>>> Claudius
>>>>>
>>>>> changes from v3:
>>>>> - ordered the transient package removal patch before the split
>>>>>
>>>>> changes from v2:
>>>>> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>>>>>
>>>>> changes from v1:
>>>>> - rebased to current next
>>>>> - added some more documentation
>>>>>
>>>>> Claudius Heine (8):
>>>>> meta: remove transient package support
>>>>> split up isar-bootstrap helper and implement pre-process pipeline
>>>>> meta/classes: add image-locales-extension class
>>>>> meta/classes: add image-account-extension class
>>>>> doc: update description of image customization
>>>>> doc: some fixes
>>>>> meta-isar: local.conf.sample: update root password and isar user
>>>>> creation
>>>>> RECIPE-API-CHANGELOG: update transient package removal + root
>>>>> password
>>>>>
>>>>> RECIPE-API-CHANGELOG.md | 22 ++
>>>>> doc/user_manual.md | 53 +++-
>>>>> meta-isar/conf/local.conf.sample | 12 +-
>>>>> .../recipes-app/example-raw/files/postinst | 13 -
>>>>> meta/classes/image-account-extension.bbclass | 257
>>>>> ++++++++++++++++++
>>>>> meta/classes/image-locales-extension.bbclass | 97 +++++++
>>>>> meta/classes/image.bbclass | 30 +-
>>>>> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
>>>>> meta/classes/rootfs.bbclass | 176 +++++++++++-
>>>>> .../isar-bootstrap/isar-bootstrap.inc | 5 +
>>>>> .../buildchroot/buildchroot-host.bb | 16 +-
>>>>> .../buildchroot/buildchroot-target.bb | 2 -
>>>>> .../buildchroot/buildchroot.inc | 26 +-
>>>>> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
>>>>> .../isar-cfg-localepurge/files/postinst | 15 -
>>>>> .../isar-cfg-localepurge.bb | 70 -----
>>>>> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
>>>>> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
>>>>> 18 files changed, 663 insertions(+), 360 deletions(-)
>>>>> create mode 100644 meta/classes/image-account-extension.bbclass
>>>>> create mode 100644 meta/classes/image-locales-extension.bbclass
>>>>> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
>>>>> delete mode 100644
>>>>> meta/recipes-support/isar-cfg-localepurge/files/postinst
>>>>> delete mode 100644
>>>>> meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
>>>>> delete mode 100644
>>>>> meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
>>>>> delete mode 100644
>>>>> meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>>>>>
>>>>
>>>>
>>>
>>
>>
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
[-- Attachment #2: local.conf --]
[-- Type: text/plain, Size: 7313 bytes --]
#
# This file is your local configuration file and is where all local user settings
# are placed. The comments in this file give some guide to the options a new user
# to the system might want to change but pretty much any configuration option can
# be set in this file. More adventurous users can look at local.conf.extended
# which contains other examples of configuration which can be placed in this file
# but new users likely won't need any of them initially.
#
# Lines starting with the '#' character are commented out and in some cases the
# default values are provided as comments to show people example syntax. Enabling
# the option is a question of removing the # character and making any change to the
# variable as required.
#
# Machine Selection
#
# You need to select a specific machine to target the build with. There are a selection
# of emulated machines available which can boot and run in the QEMU emulator:
#
# This sets the default machine to be qemuarm if no other machine is selected:
MACHINE ??= "qemuarm"
#
# Isar Configuration Selection
#
# You need to select a specific distribution configuration which will used for both:
# generation of buildchroot environment and target root filesystem.
#
# This sets the default distribution configuration:
DISTRO ??= "debian-stretch"
DISTRO_ARCH ??= "armhf"
#
# Multiple Configuration Selection
#
# If you want to use multiple configuration files for the build, list them in the
# following option.
#
# This sets the default multiple configurations used:
BBMULTICONFIG = " \
qemuarm-stretch \
qemuarm-buster \
qemuarm64-stretch \
qemuarm64-buster \
qemui386-stretch \
qemui386-buster \
qemuamd64-stretch \
bananapi-stretch \
de0-nano-soc-stretch \
hikey-stretch \
qemuamd64-buster \
qemuamd64-buster-tgz \
nand-ubi-demo-buster \
rpi-stretch \
"
#
# Where to place downloads
#
# During a first build the system will download many different source code tarballs
# from various upstream projects. This can take a while, particularly if your network
# connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you
# can preserve this directory to speed up this part of subsequent builds. This directory
# is safe to share between multiple builds on the same machine too.
#
# The default is a downloads directory under TOPDIR which is the build directory.
#
#DL_DIR ?= "${TOPDIR}/downloads"
#
# Where to place shared-state files
#
# BitBake has the capability to accelerate builds based on previously built output.
# This is done using "shared state" files which can be thought of as cache objects
# and this option determines where those files are placed.
#
# You can wipe out TMPDIR leaving this directory intact and the build would regenerate
# from these files if no changes were made to the configuration. If changes were made
# to the configuration, only shared state files where the state was still valid would
# be used (done using checksums).
#
# The default is a sstate-cache directory under TOPDIR.
#
#SSTATE_DIR ?= "${TOPDIR}/sstate-cache"
#
# Where to place the build output
#
# This option specifies where the bulk of the building work should be done and
# where BitBake should place its temporary files and output. Keep in mind that
# this includes the extraction and compilation of many applications and the toolchain
# which can use Gigabytes of hard disk space.
#
# The default is a tmp directory under TOPDIR.
#
#TMPDIR = "${TOPDIR}/tmp"
#
# Interactive shell configuration
#
# Under certain circumstances the system may need input from you and to do this it
# can launch an interactive shell. It needs to do this since the build is
# multithreaded and needs to be able to handle the case where more than one parallel
# process may require the user's attention. The default is iterate over the available
# terminal types to find one that works.
#
# Examples of the occasions this may happen are when resolving patches which cannot
# be applied, to use the devshell or the kernel menuconfig
#
# Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none
# Note: currently, Konsole support only works for KDE 3.x due to the way
# newer Konsole versions behave
#OE_TERMINAL = "auto"
# By default disable interactive patch resolution (tasks will just fail instead):
PATCHRESOLVE = "noop"
#
# Disk Space Monitoring during the build
#
# Monitor the disk space during the build. If there is less that 1GB of space or less
# than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully
# shutdown the build. If there is less that 100MB or 1K inodes, perform a hard abort
# of the build. The reason for this is that running completely out of space can corrupt
# files and damages the build in ways which may not be easily recoverable.
# It's necesary to monitor /tmp, if there is no space left the build will fail
# with very exotic errors.
BB_DISKMON_DIRS = "\
STOPTASKS,${TMPDIR},1G,100K \
STOPTASKS,${DL_DIR},1G,100K \
STOPTASKS,${SSTATE_DIR},1G,100K \
STOPTASKS,/tmp,100M,100K \
ABORT,${TMPDIR},100M,1K \
ABORT,${DL_DIR},100M,1K \
ABORT,${SSTATE_DIR},100M,1K \
ABORT,/tmp,10M,1K"
#
# Shared-state files from other locations
#
# As mentioned above, shared state files are prebuilt cache data objects which can
# used to accelerate build time. This variable can be used to configure the system
# to search other mirror locations for these objects before it builds the data itself.
#
# This can be a filesystem directory, or a remote url such as http or ftp. These
# would contain the sstate-cache results from previous builds (possibly from other
# machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the
# cache locations to check for the shared objects.
# NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH
# at the end as shown in the examples below. This will be substituted with the
# correct path within the directory structure.
#SSTATE_MIRRORS ?= "\
#file://.* http://someserver.tld/share/sstate/PATH;downloadfilename=PATH \n \
#file://.* file:///some/local/dir/sstate/PATH"
# CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to
# track the version of this file when it was generated. This can safely be ignored if
# this doesn't mean anything to you.
CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck"
#
# Enable cross-compilation support
# NOTE: this works on build host >= stretch for armhf, arm64 and amd64 targets for now.
ISAR_CROSS_COMPILE ?= "1"
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
# Set root password to 'root'
# Password was encrypted using following command:
# mkpasswd -m sha512crypt -R 10000
# mkpasswd is part of the 'whois' package of Debian
USERS += "root"
USER_root[password] ??= "$6$rounds=10000$RXeWrnFmkY$DtuS/OmsAS2cCEDo0BF5qQsizIrq6jPgXnwv3PHqREJeKd1sXdHX/ayQtuQWVDHe0KIO0/sVH8dvQm1KthF0d/"
GROUPS += "isar"
GROUP_isar[flags] = "system"
USERS += "isar"
USER_isar[gid] = "isar"
USER_isar[home] = "/var/lib/isar"
USER_isar[comment] = "My isar user"
USER_isar[flags] = "system create-home"
^ permalink raw reply [flat|nested] 30+ messages in thread
* Re: [PATCH v4 0/8] pre-processing pipeline and transient package replacement
2019-05-23 14:55 [PATCH v4 0/8] pre-processing pipeline and transient package replacement claudius.heine.ext
` (8 preceding siblings ...)
2019-05-24 12:56 ` [PATCH v4 0/8] pre-processing pipeline and transient package replacement Maxim Yu. Osipov
@ 2019-05-27 15:22 ` Maxim Yu. Osipov
9 siblings, 0 replies; 30+ messages in thread
From: Maxim Yu. Osipov @ 2019-05-27 15:22 UTC (permalink / raw)
To: claudius.heine.ext, isar-users; +Cc: Claudius Heine
On 5/23/19 4:55 PM, claudius.heine.ext@siemens.com wrote:
> From: Claudius Heine <ch@denx.de>
>
> Hi,
>
> I decided to heed Hennings advice and order the removal patch before the split
> patch. This way the transient package removal patch becomes more complete.
>
> regards,
> Claudius
>
> changes from v3:
> - ordered the transient package removal patch before the split
>
> changes from v2:
> - added 'copy-package-cache' to the ROOTFS_FEATURES in image.bbclass
>
> changes from v1:
> - rebased to current next
> - added some more documentation
Applied to the 'next',
Regards,
Maxim.
> Claudius Heine (8):
> meta: remove transient package support
> split up isar-bootstrap helper and implement pre-process pipeline
> meta/classes: add image-locales-extension class
> meta/classes: add image-account-extension class
> doc: update description of image customization
> doc: some fixes
> meta-isar: local.conf.sample: update root password and isar user
> creation
> RECIPE-API-CHANGELOG: update transient package removal + root password
>
> RECIPE-API-CHANGELOG.md | 22 ++
> doc/user_manual.md | 53 +++-
> meta-isar/conf/local.conf.sample | 12 +-
> .../recipes-app/example-raw/files/postinst | 13 -
> meta/classes/image-account-extension.bbclass | 257 ++++++++++++++++++
> meta/classes/image-locales-extension.bbclass | 97 +++++++
> meta/classes/image.bbclass | 30 +-
> meta/classes/isar-bootstrap-helper.bbclass | 143 ----------
> meta/classes/rootfs.bbclass | 176 +++++++++++-
> .../isar-bootstrap/isar-bootstrap.inc | 5 +
> .../buildchroot/buildchroot-host.bb | 16 +-
> .../buildchroot/buildchroot-target.bb | 2 -
> .../buildchroot/buildchroot.inc | 26 +-
> meta/recipes-devtools/sdkchroot/sdkchroot.bb | 48 ++--
> .../isar-cfg-localepurge/files/postinst | 15 -
> .../isar-cfg-localepurge.bb | 70 -----
> .../isar-cfg-rootpw/files/postinst.tmpl | 19 --
> .../isar-cfg-rootpw/isar-cfg-rootpw.bb | 19 --
> 18 files changed, 663 insertions(+), 360 deletions(-)
> create mode 100644 meta/classes/image-account-extension.bbclass
> create mode 100644 meta/classes/image-locales-extension.bbclass
> delete mode 100644 meta/classes/isar-bootstrap-helper.bbclass
> delete mode 100644 meta/recipes-support/isar-cfg-localepurge/files/postinst
> delete mode 100644 meta/recipes-support/isar-cfg-localepurge/isar-cfg-localepurge.bb
> delete mode 100644 meta/recipes-support/isar-cfg-rootpw/files/postinst.tmpl
> delete mode 100644 meta/recipes-support/isar-cfg-rootpw/isar-cfg-rootpw.bb
>
--
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 30+ messages in thread