Re: [PATCH v3 1/3] dpkg: move installation of dependencies into dpkg_prepare

["Maxim Yu. Osipov" <mosipov@ilbers.de>]

On 08/26/2018 09:21 AM, Cedric Hombourger wrote:
> In preparation for a rework of locking mechanisms around apt
> operations (installation of packages or inclusion of new
> packages into the isar-apt database), move the code for
> installing build dependencies out of dpkg_runbuild and into
> a new function: dpkg_prepare.

v3 series was applied to the 'next'.

Thanks,
Maxim.

> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>   doc/technical_overview.md                         | 29 ++++++++++----
>   meta/classes/dpkg-base.bbclass                    | 30 +++++++++++---
>   meta/classes/dpkg-raw.bbclass                     | 15 ++++---
>   meta/classes/dpkg.bbclass                         |  8 +++-
>   meta/recipes-devtools/buildchroot/buildchroot.inc |  9 ++++-
>   meta/recipes-devtools/buildchroot/files/build.sh  | 48 +----------------------
>   meta/recipes-devtools/buildchroot/files/common.sh | 28 +++++++++++++
>   meta/recipes-devtools/buildchroot/files/deps.sh   | 32 +++++++++++++++
>   meta/recipes-kernel/linux-module/module.inc       |  2 +-
>   9 files changed, 128 insertions(+), 73 deletions(-)
>   create mode 100644 meta/recipes-devtools/buildchroot/files/common.sh
>   create mode 100644 meta/recipes-devtools/buildchroot/files/deps.sh
> 
> diff --git a/doc/technical_overview.md b/doc/technical_overview.md
> index ddd2c79..4f6d954 100644
> --- a/doc/technical_overview.md
> +++ b/doc/technical_overview.md
> @@ -217,24 +217,37 @@ Both consist of the following steps:
>   3. Task `do_install` _only_ for `dpkg-raw`: copy all you want in your
>      debian package to `${D}`, install hooks in `${D}/DEBIAN`
>   
> -4. Task `do_build`: mount folder with unpacked files to buildchroot, execute
> -   the actual build function `dpkg_runbuild`, and finally umount again
> +4. Task `do_prepare': perform any preparation steps to the unpacked/patched
> +   sources before the build. This task calls the dpkg_prepare shell function
> +   with the buildchroot mounts in place (`dpkg_do_mounts')
>   
> -   4.1. the `dpkg_runbuild` function of `dpkg.bbclass` runs `build.sh` in the
> -        buildchroot. That performs the following:
> +   4.1. the `dpkg_prepare` function of `dpkg.bbclass` runs `/isar/deps.sh` in
> +        the buildchroot. That performs the following:
>   
>           1. Go to `/home/build/${PN}`
>   
>           2. Get list of dependencies from debian/control and install them
>   
> -        3. Run dpkg-buildpackage
> +   4.2. the `dpkg_prepare` function of `dpkg-raw.bbclass` translate the
> +        recipe meta-data into a debian/control file suitable for packaging
> +        with dpkg-deb
> +
> +5. Task `do_build`: mount folder (`dpkg_do_mounts') with unpacked files to buildchroot,
> +   execute the actual build function `dpkg_runbuild`, and finally umount again
> +   (`dpkg_undo_mounts')
> +
> +   5.1. the `dpkg_runbuild` function of `dpkg.bbclass` runs `build.sh` in the
> +        buildchroot. That performs the following:
> +
> +        1. Go to `/home/build/${PN}`
> +
> +        2. Run dpkg-buildpackage
>   
> -   4.2. the `dpkg_runbuild` function of `dpkg-raw.bbclass` basically runs
> +   5.2. the `dpkg_runbuild` function of `dpkg-raw.bbclass` basically runs
>           `dpkg-deb` to construct a Debian package from a folder of files,
>   	without compiling anything
>   
> -
> -5. Task `do_deploy_deb`: install successfully built packages
> +6. Task `do_deploy_deb`: install successfully built packages
>      `${WORKDIR}/*.deb` to deploy directory `${DEPLOY_DIR_DEB}`
>   
>   ## 3.6 Populate Target Filesystem
> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
> index a41df2c..5826357 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -13,7 +13,7 @@ python __anonymous() {
>           dep = "buildchroot-host:do_build"
>           rootfs = d.getVar('BUILDCHROOT_HOST_DIR', True)
>   
> -    d.setVarFlag('do_build', 'depends', dep)
> +    d.setVarFlag('do_prepare', 'depends', dep)
>       d.setVar('BUILDCHROOT_DIR', rootfs)
>   }
>   
> @@ -32,7 +32,7 @@ addtask patch after do_adjust_git before do_build
>   
>   # Add dependency between Isar recipes
>   DEPENDS ?= ""
> -do_build[deptask] = "do_deploy_deb"
> +do_prepare[deptask] = "do_deploy_deb"
>   
>   def get_package_srcdir(d):
>       s = d.getVar("S", True)
> @@ -50,7 +50,10 @@ PPS ?= "${@get_package_srcdir(d)}"
>   BUILDROOT = "${BUILDCHROOT_DIR}/${PP}"
>   do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>   
> -# default to "emtpy" implementation
> +# default to "emtpy" implementation for dpkg_prepare() and dpkg_runbuild()
> +dpkg_prepare() {
> +    true
> +}
>   dpkg_runbuild() {
>       die "This should never be called, overwrite it in your derived class"
>   }
> @@ -58,7 +61,7 @@ dpkg_runbuild() {
>   MOUNT_LOCKFILE = "${BUILDCHROOT_DIR}/mount.lock"
>   
>   # Wrap the function dpkg_runbuild with the bind mount for buildroot
> -do_build() {
> +dpkg_do_mounts() {
>       mkdir -p ${BUILDROOT}
>       sudo mount --bind ${WORKDIR} ${BUILDROOT}
>   
> @@ -70,13 +73,28 @@ do_build() {
>               mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${BUILDCHROOT_DIR}/dev
>               mount -t proc none ${BUILDCHROOT_DIR}/proc
>           fi'
> +}
>   
> -    dpkg_runbuild
> -
> +dpkg_undo_mounts() {
>       sudo umount ${BUILDROOT} 2>/dev/null || true
>       sudo rmdir ${BUILDROOT} 2>/dev/null || true
>   }
>   
> +do_prepare() {
> +    dpkg_do_mounts
> +    dpkg_prepare
> +    dpkg_undo_mounts
> +}
> +
> +addtask prepare after do_patch before do_build
> +do_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> +
> +do_build() {
> +    dpkg_do_mounts
> +    dpkg_runbuild
> +    dpkg_undo_mounts
> +}
> +
>   CLEANFUNCS += "repo_clean"
>   
>   repo_clean() {
> diff --git a/meta/classes/dpkg-raw.bbclass b/meta/classes/dpkg-raw.bbclass
> index 28233ac..c5f1c88 100644
> --- a/meta/classes/dpkg-raw.bbclass
> +++ b/meta/classes/dpkg-raw.bbclass
> @@ -14,9 +14,9 @@ do_install() {
>   }
>   
>   do_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -addtask install after do_unpack before do_deb_package_prepare
> +addtask install after do_unpack before do_prepare
>   
> -do_deb_package_prepare() {
> +deb_package_prepare() {
>   	sudo rm -rf ${D}/DEBIAN
>   	mkdir -p ${D}/DEBIAN
>   	cat<<-__EOF__ > ${D}/DEBIAN/control
> @@ -42,17 +42,16 @@ do_deb_package_prepare() {
>   	done
>   }
>   
> -do_deb_package_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -addtask deb_package_prepare after do_install before do_deb_package_conffiles
> -
> -do_deb_package_conffiles() {
> +deb_package_conffiles() {
>   	CONFFILES=${D}/DEBIAN/conffiles
>   	find ${D} -type f -path '${D}/etc/*' | sed -e 's|^${D}|/|' >> $CONFFILES
>   	test -s $CONFFILES || rm $CONFFILES
>   }
>   
> -do_deb_package_conffiles[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> -addtask deb_package_conffiles after do_deb_package_prepare before do_build
> +dpkg_prepare() {
> +	deb_package_prepare
> +	deb_package_conffiles
> +}
>   
>   dpkg_runbuild() {
>   	sudo chown -R root:root ${D}/DEBIAN/
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index ab70645..e8bd6ba 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -3,8 +3,14 @@
>   
>   inherit dpkg-base
>   
> +# Install build dependencies for package
> +dpkg_prepare() {
> +    E="${@ bb.utils.export_proxies(d)}"
> +    sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
> +}
> +
>   # Build package from sources using build script
>   dpkg_runbuild() {
>       E="${@ bb.utils.export_proxies(d)}"
> -    sudo -E chroot ${BUILDCHROOT_DIR} /build.sh ${PP}/${PPS} ${DISTRO_ARCH}
> +    sudo -E chroot ${BUILDCHROOT_DIR} /isar/build.sh ${PP}/${PPS} ${DISTRO_ARCH}
>   }
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
> index 43e3cd6..7dd909e 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
> @@ -8,7 +8,9 @@ LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260
>   
>   FILESPATH_prepend := "${THISDIR}/files:"
>   SRC_URI = "file://configscript.sh \
> -           file://build.sh"
> +           file://build.sh \
> +           file://common.sh \
> +           file://deps.sh"
>   PV = "1.0"
>   
>   inherit isar-bootstrap-helper
> @@ -27,7 +29,10 @@ do_build() {
>   
>       # Install package builder script
>       sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
> -    sudo install -m 755 ${WORKDIR}/build.sh ${BUILDCHROOT_DIR}
> +    sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
> +    sudo install -m 755 ${WORKDIR}/build.sh ${BUILDCHROOT_DIR}/isar/
> +    sudo install -m 755 ${WORKDIR}/common.sh ${BUILDCHROOT_DIR}/isar/
> +    sudo install -m 755 ${WORKDIR}/deps.sh ${BUILDCHROOT_DIR}/isar/
>   
>       # Configure root filesystem
>       sudo install -m 755 ${WORKDIR}/configscript.sh ${BUILDCHROOT_DIR}
> diff --git a/meta/recipes-devtools/buildchroot/files/build.sh b/meta/recipes-devtools/buildchroot/files/build.sh
> index e2dabab..3c80bd7 100644
> --- a/meta/recipes-devtools/buildchroot/files/build.sh
> +++ b/meta/recipes-devtools/buildchroot/files/build.sh
> @@ -4,53 +4,7 @@
>   # Copyright (C) 2015-2017 ilbers GmbH
>   # Copyright (c) 2018 Siemens AG
>   
> -set -e
> -
> -# Create human-readable names
> -target_arch=$2
> -
> -# Notes:
> -#   mk-build-deps for jessie and jtretch has different parameter name to specify
> -#   host architecture.
> -debian_version=$(cut -c1 /etc/debian_version)
> -if [ $(($debian_version)) -ge 9 ]; then
> -    set_arch="--host-arch $target_arch"
> -else
> -    set_arch="-a $target_arch"
> -fi
> -
> -# Go to build directory
> -cd $1
> -
> -# To avoid Perl locale warnings:
> -export LC_ALL=C
> -export LANG=C
> -export LANGUAGE=C
> -
> -# Install command to be used by mk-build-deps
> -# Notes:
> -#   1) everything before the -y switch is unchanged from the defaults
> -#   2) we add -y to go non-interactive
> -install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"
> -
> -(
> -    # Lock-protected because apt and dpkg do not wait in case of contention
> -    flock 42 || exit 1
> -
> -    # Make sure that we have latest isar-apt content.
> -    # Options meaning:
> -    #   Dir::Etc::sourcelist - specifies which source to be used
> -    #   Dir::Etc::sourceparts - disables looking for the other sources
> -    #   APT::Get::List-Cleanup - do not erase obsolete packages list for
> -    #                            upstream in '/var/lib/apt/lists'
> -    apt-get update \
> -        -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
> -        -o Dir::Etc::sourceparts="-" \
> -        -o APT::Get::List-Cleanup="0"
> -
> -    # Install all build deps
> -    mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
> -) 42>/dpkg.lock
> +source /isar/common.sh
>   
>   # If autotools files have been created, update their timestamp to
>   # prevent them from being regenerated
> diff --git a/meta/recipes-devtools/buildchroot/files/common.sh b/meta/recipes-devtools/buildchroot/files/common.sh
> new file mode 100644
> index 0000000..b7551eb
> --- /dev/null
> +++ b/meta/recipes-devtools/buildchroot/files/common.sh
> @@ -0,0 +1,28 @@
> +#!/bin/bash
> +#
> +# This software is a part of ISAR.
> +# Copyright (C) 2015-2017 ilbers GmbH
> +# Copyright (c) 2018 Siemens AG
> +
> +set -e
> +
> +# Create human-readable names
> +target_arch=$2
> +
> +# Notes:
> +#   mk-build-deps for jessie and jtretch has different parameter name to specify
> +#   host architecture.
> +debian_version=$(cut -c1 /etc/debian_version)
> +if [ $(($debian_version)) -ge 9 ]; then
> +    set_arch="--host-arch $target_arch"
> +else
> +    set_arch="-a $target_arch"
> +fi
> +
> +# Go to build directory
> +cd $1
> +
> +# To avoid Perl locale warnings:
> +export LC_ALL=C
> +export LANG=C
> +export LANGUAGE=C
> diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
> new file mode 100644
> index 0000000..854a4d5
> --- /dev/null
> +++ b/meta/recipes-devtools/buildchroot/files/deps.sh
> @@ -0,0 +1,32 @@
> +#!/bin/bash
> +#
> +# This software is a part of ISAR.
> +# Copyright (C) 2015-2017 ilbers GmbH
> +# Copyright (c) 2018 Siemens AG
> +
> +source /isar/common.sh
> +
> +# Install command to be used by mk-build-deps
> +# Notes:
> +#   1) everything before the -y switch is unchanged from the defaults
> +#   2) we add -y to go non-interactive
> +install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"
> +
> +(
> +    # Lock-protected because apt and dpkg do not wait in case of contention
> +    flock 42 || exit 1
> +
> +    # Make sure that we have latest isar-apt content.
> +    # Options meaning:
> +    #   Dir::Etc::sourcelist - specifies which source to be used
> +    #   Dir::Etc::sourceparts - disables looking for the other sources
> +    #   APT::Get::List-Cleanup - do not erase obsolete packages list for
> +    #                            upstream in '/var/lib/apt/lists'
> +    apt-get update \
> +        -o Dir::Etc::sourcelist="sources.list.d/isar-apt.list" \
> +        -o Dir::Etc::sourceparts="-" \
> +        -o APT::Get::List-Cleanup="0"
> +
> +    # Install all build deps
> +    mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
> +) 42>/dpkg.lock
> diff --git a/meta/recipes-kernel/linux-module/module.inc b/meta/recipes-kernel/linux-module/module.inc
> index 3075f44..3a3cab1 100644
> --- a/meta/recipes-kernel/linux-module/module.inc
> +++ b/meta/recipes-kernel/linux-module/module.inc
> @@ -17,7 +17,7 @@ AUTOLOAD ?= "0"
>   
>   inherit dpkg
>   
> -dpkg_runbuild_prepend() {
> +do_prepare_prepend() {
>       cp -r ${WORKDIR}/debian ${S}/
>       sed -i -e 's/@PN@/${PN}/g' -e 's/@PV@/${PV}/g' \
>              -e 's/@KERNEL_NAME@/${KERNEL_NAME}/g' \
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov