Re: [PATCH 1/2] meta: Support for ccache for custom packages in buildchroot

[Jan Kiszka <jan.kiszka@siemens.com>]

On 13.09.21 17:10, Uladzimir Bely wrote:
> Add `INHERIT += "ccache"` to local.conf or package recipe
> to globally activate the ccache functionality.
> Add `CCACHE_DISABLE = '1'` to the recipe to disable ccache for it.
> 
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> ---
>  meta-isar/conf/local.conf.sample              |  5 ++++
>  meta/classes/buildchroot.bbclass              |  6 +++++
>  meta/classes/ccache.bbclass                   | 23 +++++++++++++++++++
>  meta/classes/dpkg.bbclass                     | 12 +++++++++-
>  .../buildchroot/buildchroot.inc               |  1 +
>  .../buildchroot/files/build.sh                |  6 +++++
>  .../buildchroot/files/common.sh               |  1 +
>  7 files changed, 53 insertions(+), 1 deletion(-)
>  create mode 100644 meta/classes/ccache.bbclass
> 
> diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
> index 96a8beb..0f1a030 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -222,3 +222,8 @@ USER_isar[flags] += "clear-text-password"
>  
>  # Uncomment the below line to debug WIC.
>  # WIC_CREATE_EXTRA_ARGS += "-D"
> +
> +# Uncomment this to use ccache for custom packages
> +#INHERIT += "ccache"
> +# Uncomment and set own top level ccache directory to share between builds
> +#CCACHE_TOP_DIR ?= "${TMPDIR}/ccache"
> diff --git a/meta/classes/buildchroot.bbclass b/meta/classes/buildchroot.bbclass
> index e9eb9af..598458d 100644
> --- a/meta/classes/buildchroot.bbclass
> +++ b/meta/classes/buildchroot.bbclass
> @@ -32,6 +32,12 @@ buildchroot_do_mounts() {
>              mount --bind '${REPO_ISAR_DIR}/${DISTRO}' '${BUILDCHROOT_DIR}/isar-apt'
>          mountpoint -q '${BUILDCHROOT_DIR}/downloads' ||
>              mount --bind '${DL_DIR}' '${BUILDCHROOT_DIR}/downloads'
> +        if [ "${@repr(bb.utils.to_boolean(d.getVar('CCACHE_BUILDCHROOT')))}" = 'True' ]

Why so complex? Elsewhere, we just do

	if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then

and are done.

> +        then
> +            mkdir -p '${BUILDCHROOT_DIR}/ccache'
> +            mountpoint -q '${BUILDCHROOT_DIR}/ccache' ||
> +                mount --bind '${CCACHE_DIR}' '${BUILDCHROOT_DIR}/ccache'
> +        fi
>          mountpoint -q '${BUILDCHROOT_DIR}/dev' ||
>              mount --rbind /dev '${BUILDCHROOT_DIR}/dev'
>          mount --make-rslave '${BUILDCHROOT_DIR}/dev'
> diff --git a/meta/classes/ccache.bbclass b/meta/classes/ccache.bbclass
> new file mode 100644
> index 0000000..2259417
> --- /dev/null
> +++ b/meta/classes/ccache.bbclass
> @@ -0,0 +1,23 @@
> +#
> +# Usage:
> +# - Enable ccache
> +#   Add the following line to a conffile such as conf/local.conf:
> +#   INHERIT += "ccache"
> +#
> +# - Disable ccache for a recipe
> +#   Add the following line to the recipe if it can't be built with ccache:
> +#   CCACHE_DISABLE = '1'
> +#
> +# - Share ccache files between different builds
> +#   Set CCACHE_TOP_DIR to a shared dir
> +#   CCACHE_TOP_DIR = /path/to/shared_ccache/
> +
> +CCACHE_TOP_DIR ?= "${TMPDIR}/ccache"
> +CCACHE_DIR ?= "${CCACHE_TOP_DIR}/${DISTRO}-${MACHINE}"
> +
> +CCACHE_BUILDCHROOT ?= "1"
> +
> +do_init_ccache[dirs] = "${CCACHE_DIR}"
> +do_init_ccache() {
> +}
> +addtask do_init_ccache before do_rootfs
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index 4e7c2f7..af2ba2e 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -32,6 +32,16 @@ addtask devshell after do_install_builddeps
>  dpkg_runbuild() {
>      E="${@ isar_export_proxies(d)}"
>      export PARALLEL_MAKE="${PARALLEL_MAKE}"
> +    if [ "${@repr(bb.utils.to_boolean(d.getVar('CCACHE_BUILDCHROOT')))}" = 'True' ]
> +    then
> +        USE_CCACHE="1"
> +    else
> +        USE_CCACHE="0"
> +    fi
> +    if [ "${@repr(bb.utils.to_boolean(d.getVar('CCACHE_DISABLE')))}" = 'True' ]
> +    then
> +        USE_CCACHE="0"
> +    fi

Way to convoluted as I already commented on patch 2. Just use a single
ISAR_CCACHE_ENABLE test here. That var can be set globally (local.conf &
Co) and overwritten locally (recipes).

>      sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
> -         /isar/build.sh ${PP}/${PPS} ${PACKAGE_ARCH}
> +         /isar/build.sh ${PP}/${PPS} ${PACKAGE_ARCH} ${USE_CCACHE}
>  }
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
> index 31524a1..726c7bb 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.inc
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
> @@ -32,6 +32,7 @@ BUILDCHROOT_PREINSTALL_COMMON = " \
>      make \
>      debhelper \
>      autotools-dev \
> +    ccache \
>      dpkg \
>      locales \
>      docbook-to-man \
> diff --git a/meta/recipes-devtools/buildchroot/files/build.sh b/meta/recipes-devtools/buildchroot/files/build.sh
> index 101581d..b642f42 100644
> --- a/meta/recipes-devtools/buildchroot/files/build.sh
> +++ b/meta/recipes-devtools/buildchroot/files/build.sh
> @@ -14,4 +14,10 @@ for i in configure aclocal.m4 Makefile.am Makefile.in; do
>      fi
>  done
>  
> +if [ "$use_ccache" == "1" ]
> +then
> +    export CCACHE_DIR=/ccache
> +    export PATH=/usr/lib/ccache:$PATH
> +fi

You can do that unconditionally. That will also save you from passing
use_ccache down.

> +
>  ${GBP_PREFIX}dpkg-buildpackage -a$target_arch -d --source-option=-I
> diff --git a/meta/recipes-devtools/buildchroot/files/common.sh b/meta/recipes-devtools/buildchroot/files/common.sh
> index 0063a38..7085f44 100644
> --- a/meta/recipes-devtools/buildchroot/files/common.sh
> +++ b/meta/recipes-devtools/buildchroot/files/common.sh
> @@ -17,6 +17,7 @@ fi
>  
>  # Create human-readable names
>  target_arch=$2
> +use_ccache=$3
>  
>  set_arch="--host-arch $target_arch"
>  
> 

In general, a very nice feature! I'm looking forward to much faster
kernel builds when only adding a patch on top of flipping a simple
feature. We just need to tune the interface, and the implementation can
become even more compact.

Jan

-- 
Siemens AG, T RDA IOT
Corporate Competence Center Embedded Linux