[PATCH 2/3] linux-custom: Split up binaries from kernel headers to kbuild packages

["Koch, Stefan" <stefan-koch@siemens.com>]

When using a cross build this patch does introduce
host and target specific kernel kbuild packages that
ship the "scripts" and "tools" binaries.
The kernel headers fulfill this using symlinks to point
to the "scripts" and "tools" of the respective kernel kbuild package.

Known from doc/custom_kernel.inc:
- The kernel headers package has not supported both native
  and cross compilation of kernel modules when itself was cross built
- Future roadmap: Generate kernel headers package for both host
  and target when using a cross build

Known from debian kernel packages structure:
- Generate a kernel headers package without binaries
- Create specific kernel kbuild packages that
  ship the "scripts" and "tools" binaries
- Using symlinks to point to the "scripts"
  and "tools" binaries

Signed-off-by: Stefan Koch <stefan-koch@siemens.com>
---
 .../linux/files/debian/control.tmpl           |  18 ++-
 .../linux/files/debian/isar/build.tmpl        |   9 +-
 .../linux/files/debian/isar/common.tmpl       |   7 +-
 .../linux/files/debian/isar/install.tmpl      |  64 +++++++++--
 .../linux/files/debian/rules.tmpl             |   5 +-
 meta/recipes-kernel/linux/linux-custom.inc    | 104 ++++++++++++++++--
 6 files changed, 178 insertions(+), 29 deletions(-)

diff --git a/meta/recipes-kernel/linux/files/debian/control.tmpl b/meta/recipes-kernel/linux/files/debian/control.tmpl
index dd0b624..adac4ef 100644
--- a/meta/recipes-kernel/linux/files/debian/control.tmpl
+++ b/meta/recipes-kernel/linux/files/debian/control.tmpl
@@ -26,7 +26,7 @@ Section: devel
 Provides: linux-kernel-headers
 Architecture: any
 Description: Linux support headers for userspace development
- This package provides userspaces headers from the Linux kernel.  These headers
+ This package provides userspaces headers from the Linux kernel. These headers
  are used by the installed headers for GNU glibc and other system libraries.
 
 Package: linux-image-${KERNEL_NAME_PROVIDED}-dbg
@@ -35,3 +35,19 @@ Architecture: any
 Description: Linux kernel debugging symbols for @KR@
  This package will come in handy if you need to debug the kernel. It provides
  all the necessary debug symbols for the kernel and its modules.
+
+Package: linux-kbuild-${KERNEL_NAME_PROVIDED}
+Architecture: any
+Depends: ${KERNEL_HEADERS_DEBIAN_DEPENDS}, ${perl:Depends}, ${shlib:Depends}
+Description: ${KERNEL_NAME_PROVIDED} Linux kbuild scripts and tools for @KR@
+ This package provides kernel kbuild scripts and tools for @KR@
+ This is useful for people who need to build external modules
+
+Package: linux-kbuild-${KERNEL_NAME_PROVIDED}-cross
+Architecture: ${CONTROL_TARGET_ARCH}
+Depends: ${KERNEL_HEADERS_DEBIAN_DEPENDS}, ${perl:Depends}, ${shlib:Depends}
+Description: ${KERNEL_NAME_PROVIDED} Linux kbuild scripts and tools for @KR@
+ This package provides kernel kbuild scripts and tools
+ as ${HOST_ARCH} cross binaries for @KR@
+ This is useful for those who need to cross build
+ external modules using ISAR's sbuild-chroot-host
diff --git a/meta/recipes-kernel/linux/files/debian/isar/build.tmpl b/meta/recipes-kernel/linux/files/debian/isar/build.tmpl
index 94cfbe0..7e2daad 100644
--- a/meta/recipes-kernel/linux/files/debian/isar/build.tmpl
+++ b/meta/recipes-kernel/linux/files/debian/isar/build.tmpl
@@ -13,16 +13,19 @@ do_build() {
     # Trace what we do here
     set -x
 
+    # Set correct build directory for kernel or kbuild (kernel scripts and tools) cases
+    build_dir=${KERNEL_BUILD_DIR}
+
     # Process existing kernel configuration to make sure it is complete
     # (use defaults for options that were not specified)
-    ${MAKE} O=${KERNEL_BUILD_DIR} olddefconfig prepare
+    ${MAKE} O=${build_dir} olddefconfig prepare
 
     # Transfer effective kernel version into control file and scripts
-    KR=$(${MAKE} O=${KERNEL_BUILD_DIR} -s --no-print-directory kernelrelease)
+    KR=$(${MAKE} O=${build_dir} -s --no-print-directory kernelrelease)
     sed -i "s/@KR@/${KR}/g" ${S}/debian/control ${S}/debian/linux-image-${KERNEL_NAME_PROVIDED}.*
 
     # Build the Linux kernel
-    ${MAKE} O=${KERNEL_BUILD_DIR} ${PARALLEL_MAKE} KCFLAGS="${KCFLAGS}"
+    ${MAKE} O=${build_dir} ${PARALLEL_MAKE} KCFLAGS="${KCFLAGS}"
 
     # Stop tracing
     set +x
diff --git a/meta/recipes-kernel/linux/files/debian/isar/common.tmpl b/meta/recipes-kernel/linux/files/debian/isar/common.tmpl
index 52ebebb..55d6123 100644
--- a/meta/recipes-kernel/linux/files/debian/isar/common.tmpl
+++ b/meta/recipes-kernel/linux/files/debian/isar/common.tmpl
@@ -5,10 +5,14 @@
 set -e
 
 # Isar settings
-ARCH=${KERNEL_ARCH}
 KERNEL_PKG_IMAGE=linux-image-${KERNEL_NAME_PROVIDED}
 KERNEL_PKG_KERN_HEADERS=linux-headers-${KERNEL_NAME_PROVIDED}
 KERNEL_PKG_LIBC_HEADERS=linux-libc-dev
+KERNEL_PKG_KERN_KBUILD=linux-kbuild-${KERNEL_NAME_PROVIDED}
+
+if [ -z "${ARCH}" ]; then
+	ARCH=${KERNEL_ARCH}
+fi
 
 # Constants
 KCONF=.config
@@ -19,6 +23,7 @@ deb_img_dir=${deb_top_dir}/${KERNEL_PKG_IMAGE}
 deb_dbg_dir=${deb_img_dir}-dbg
 deb_kern_hdr_dir=${deb_top_dir}/${KERNEL_PKG_KERN_HEADERS}
 deb_libc_hdr_dir=${deb_top_dir}/${KERNEL_PKG_LIBC_HEADERS}
+deb_kern_kbuild_dir=${deb_top_dir}/${KERNEL_PKG_KERN_KBUILD}
 
 # Array of packages to be generated
 declare -A kern_pkgs
diff --git a/meta/recipes-kernel/linux/files/debian/isar/install.tmpl b/meta/recipes-kernel/linux/files/debian/isar/install.tmpl
index 0a8645d..890a996 100644
--- a/meta/recipes-kernel/linux/files/debian/isar/install.tmpl
+++ b/meta/recipes-kernel/linux/files/debian/isar/install.tmpl
@@ -33,6 +33,7 @@ do_install() {
     # Trace what we do here
     set -x
 
+    # Run the install steps
     install_image
     if [ "${ARCH}" != "um" ]; then
         install_config
@@ -43,6 +44,19 @@ do_install() {
     install_kmods
     install_headers
 
+    if [ -d "$(dirname ${O})/build-full-kbuild-target" ]; then
+        # Install cross kernel scripts and tools
+        install_kbuild ${deb_kern_kbuild_dir}-cross build-full
+
+        # Cleanup and install kernel scripts and tools for target arch
+        rm -rf ${deb_kern_kbuild_dir}
+        install_kbuild ${deb_kern_kbuild_dir} build-full-kbuild-target
+    else
+       # Cleanup and install kernel scripts and tools
+       rm -rf ${deb_kern_kbuild_dir}
+       install_kbuild ${deb_kern_kbuild_dir} build-full
+    fi
+
     # Stop tracing
     set +x
 }
@@ -168,21 +182,15 @@ kernel_headers() {
     mkdir -p ${destdir}
     mkdir -p ${deb_kern_hdr_dir}/lib/modules/${krel}
 
-    (cd ${S}; find . -name 'Makefile*' -o -name 'Kconfig*' -o -name '*.pl') >>${src_hdr_files}
-    (cd ${S}; find arch/*/include include scripts -type f -o -type l) >>${src_hdr_files}
+    (cd ${S}; find . -not -path './scripts/*' -a -not -path './tools/*' -a \( -name 'Makefile*' -o -name 'Kconfig*' -o -name '*.pl' \)) >>${src_hdr_files}
+    (cd ${S}; find arch/*/include include -type f -o -type l) >>${src_hdr_files}
     (cd ${S}; find arch/${ARCH} -name module.lds -o -name Kbuild.platforms -o -name Platform) >>${src_hdr_files}
     (cd ${S}; find $(find arch/${ARCH} -name include -o -name scripts -type d) -type f) >>${src_hdr_files}
 
     if [ -n "${CONFIG_MODULES}" ]; then
         echo Module.symvers >> ${obj_hdr_files}
     fi
-    (cd ${O}; find arch/${ARCH}/include include scripts -type f) >>${obj_hdr_files}
-    if [ -n "${CONFIG_STACK_VALIDATION}" ]; then
-        (cd ${O}; find tools/objtool -type f -executable) >>${obj_hdr_files}
-    fi
-    if [ -n "${CONFIG_GCC_PLUGINS}" ]; then
-        (cd ${O}; find scripts/gcc-plugins -name *.so -o -name gcc-common.h) >>${obj_hdr_files}
-    fi
+    (cd ${O}; find arch/${ARCH}/include include -type f) >>${obj_hdr_files}
 
     # deploy files that were matched above
     tar -C ${S} -cf - -T - <${src_hdr_files} | tar -C ${destdir} -xf -
@@ -191,8 +199,11 @@ kernel_headers() {
     # add the kernel config
     cp ${O}/${KCONF} ${destdir}/.config
 
-    # handle kernel development tools
-    kernel_tools
+    # add symlink to scripts and tools directories
+    ln -sf ../../lib/linux-kbuild-${krel}/scripts ${destdir}/scripts
+    if [ -n "${CONFIG_STACK_VALIDATION}" ]; then
+        ln -sf ../../lib/linux-kbuild-${krel}/tools ${destdir}/tools
+    fi
 
     # create symlinks
     ln -sf /${kernel_headers_dir} ${deb_kern_hdr_dir}/lib/modules/${krel}/build
@@ -206,4 +217,35 @@ install_headers() {
     kernel_headers
 }
 
+install_kbuild() {
+    kernel_kbuild_dir=usr/lib/linux-kbuild-${krel}
+    destdir=${1}/${kernel_kbuild_dir}
+    src_kbuild_files=$(mktemp)
+    obj_kbuild_files=$(mktemp)
+
+    if [ -n "${2}" ]; then
+        O=$(dirname ${O})/${2}
+    fi
+
+    mkdir -p ${destdir}
+
+    (cd ${S}; find . -path './scripts/*' -a -path './tools/*' -a \( -name 'Makefile*' -o -name 'Kconfig*' -o -name '*.pl' \)) >>${src_kbuild_files}
+    (cd ${S}; find scripts -type f -o -type l) >>${src_kbuild_files}
+
+    (cd ${O}; find scripts -type f) >>${obj_kbuild_files}
+    if [ -n "${CONFIG_STACK_VALIDATION}" ]; then
+        (cd ${O}; find tools/objtool -type f -executable) >>${obj_kbuild_files}
+    fi
+    if [ -n "${CONFIG_GCC_PLUGINS}" ]; then
+        (cd ${O}; find scripts/gcc-plugins -name *.so -o -name gcc-common.h) >>${obj_kbuild_files}
+    fi
+
+    # deploy files that were matched above
+    tar -C ${S} -cf - -T - <${src_kbuild_files} | tar -C ${destdir} -xf -
+    tar -C ${O} -cf - -T - <${obj_kbuild_files} | tar -C ${destdir} -xf -
+
+    # handle kernel development tools
+    kernel_tools
+}
+
 main install ${*}
diff --git a/meta/recipes-kernel/linux/files/debian/rules.tmpl b/meta/recipes-kernel/linux/files/debian/rules.tmpl
index 8063c49..e131288 100755
--- a/meta/recipes-kernel/linux/files/debian/rules.tmpl
+++ b/meta/recipes-kernel/linux/files/debian/rules.tmpl
@@ -36,4 +36,7 @@ override_dh_auto_test:
 	true
 
 override_dh_strip:
-	unset DEB_HOST_GNU_TYPE && dh_strip -Xvmlinu --no-automatic-dbgsym
+	dh_strip -Xvmlinu -Xlinux-kbuild-${KERNEL_NAME_PROVIDED}-cross --no-automatic-dbgsym
+
+override_dh_shlibdeps:
+	dh_shlibdeps -Xlinux-kbuild-${KERNEL_NAME_PROVIDED}
diff --git a/meta/recipes-kernel/linux/linux-custom.inc b/meta/recipes-kernel/linux/linux-custom.inc
index 96f0afc..0f8ba50 100644
--- a/meta/recipes-kernel/linux/linux-custom.inc
+++ b/meta/recipes-kernel/linux/linux-custom.inc
@@ -74,12 +74,14 @@ TEMPLATE_VARS += "                \
     KERNEL_ARCH                   \
     KERNEL_DEBIAN_DEPENDS         \
     KERNEL_BUILD_DIR              \
+    KERNEL_KBUILD_DIR             \
     KERNEL_FILE                   \
     KERNEL_HEADERS_DEBIAN_DEPENDS \
     LINUX_VERSION_EXTENSION       \
     KERNEL_NAME_PROVIDED          \
     KERNEL_CONFIG_FRAGMENTS       \
     KCFLAGS                       \
+    CONTROL_TARGET_ARCH           \
 "
 
 inherit dpkg
@@ -91,30 +93,54 @@ KCFLAGS ?= "-fdebug-prefix-map=${CURDIR}=."
 # Derive name of the kernel packages from the name of this recipe
 KERNEL_NAME_PROVIDED ?= "${@ d.getVar('PN', True).partition('linux-')[2]}"
 
-# Make bitbake know we will be producing linux-image and linux-headers packages
 python() {
     kernel_name = d.getVar("KERNEL_NAME_PROVIDED", True)
-    d.setVar('PROVIDES', 'linux-image-' + kernel_name + ' ' + \
-                         'linux-headers-' + kernel_name)
+    distro_arch = d.getVar("DISTRO_ARCH", True)
+    host_arch = d.getVar("HOST_ARCH", True)
+    target_arch = ""
+    kbuild_depends = "linux-kbuild-" + kernel_name
+
+    # For different distro and host archs
+    # - Set target arch (empty string disables package build)
+    # - Add dependency for sbuild-chroot-target
+    #   to allow building arch specific kbuild scripts and tools
+    # - Set correct name of kbuild package dependency
+    if distro_arch != host_arch:
+        d.appendVar("SCHROOT_DEP", " sbuild-chroot-target:do_build")
+        if d.getVar("ISAR_CROSS_COMPILE", True) == "1":
+            target_arch = distro_arch
+            kbuild_depends = kbuild_depends + "-cross | " + kbuild_depends
+
+    # Set CONTROL_TARGET_ARCH
+    d.setVar("CONTROL_TARGET_ARCH", target_arch)
+
+    # Make bitbake know we will be producing
+    # linux-image and linux-headers packages
+    d.setVar("PROVIDES", "linux-image-" + kernel_name + " " + \
+                         "linux-headers-" + kernel_name)
+
+    # Set dependency for kbuild package
+    d.appendVar("KERNEL_HEADERS_DEBIAN_DEPENDS", kbuild_depends)
 }
 
-def get_kernel_arch(d):
-    distro_arch = d.getVar("DISTRO_ARCH")
-    if distro_arch in ["amd64", "i386"]:
+def get_kernel_arch(d, arch="DISTRO_ARCH"):
+    arch = d.getVar(arch)
+    if arch in ["amd64", "i386"]:
         kernel_arch = "x86"
-    elif distro_arch == "arm64":
+    elif arch == "arm64":
         kernel_arch = "arm64"
-    elif distro_arch == "armhf":
+    elif arch == "armhf":
         kernel_arch = "arm"
-    elif distro_arch == "mipsel":
+    elif arch == "mipsel":
         kernel_arch = "mips"
-    elif distro_arch == "riscv64":
+    elif arch == "riscv64":
         kernel_arch = "riscv"
     else:
         kernel_arch = ""
     return kernel_arch
 
 KERNEL_ARCH ??= "${@get_kernel_arch(d)}"
+KERNEL_HOST_ARCH ??= "${@get_kernel_arch(d, 'HOST_ARCH')}"
 
 def config_fragments(d):
     fragments = []
@@ -157,6 +183,7 @@ do_prepare_build_prepend() {
 
 # build directory for our "full" kernel build
 KERNEL_BUILD_DIR = "build-full"
+KERNEL_KBUILD_DIR = "${KERNEL_BUILD_DIR}-kbuild"
 
 def get_kernel_config_target(d):
     kernel_defconfig = d.getVar('KERNEL_DEFCONFIG', True)
@@ -187,15 +214,19 @@ def get_kernel_config_fragments(d):
 KERNEL_CONFIG_FRAGMENTS = "${@get_kernel_config_fragments(d)}"
 
 dpkg_configure_kernel() {
+	build_dir="${KERNEL_BUILD_DIR}"
+	if [ -n "${1}" ]; then
+		build_dir="${1}"
+	fi
 	grep -q "KERNEL_CONFIG_TARGET=" ${S}/debian/isar/configure ||
 		cat << EOF | sed -i '/^do_configure() {/ r /dev/stdin' ${S}/debian/isar/configure
     KERNEL_CONFIG_TARGET="${@get_kernel_config_target(d)}"
 EOF
 
-	rm -rf ${S}/${KERNEL_BUILD_DIR} && mkdir -p ${S}/${KERNEL_BUILD_DIR}
+	rm -rf ${S}/${build_dir} && mkdir -p ${S}/${build_dir}
 	if [ -n "${KERNEL_DEFCONFIG}" ]; then
 		if [ -e "${WORKDIR}/${KERNEL_DEFCONFIG}" ]; then
-			cp ${WORKDIR}/${KERNEL_DEFCONFIG} ${S}/${KERNEL_BUILD_DIR}/.config
+			cp ${WORKDIR}/${KERNEL_DEFCONFIG} ${S}/${build_dir}/.config
 		fi
 	fi
 
@@ -210,4 +241,53 @@ EOF
 
 dpkg_runbuild_prepend() {
 	dpkg_configure_kernel
+
+	if [ "${ISAR_CROSS_COMPILE}" = "1" ] && [ "${HOST_ARCH}" != "${DISTRO_ARCH}" ]; then
+		# Create temporarily kernel schroot-target configuration
+		schroot_create_configs ${SCHROOT_TARGET_DIR}
+
+		# Begin kernel schroot-target session
+		schroot_session=$(schroot -b -c ${SBUILD_CHROOT})
+
+		# Install build deps for schroot session
+		schroot -u root -r -c $schroot_session -- sh -c " \
+			cd ${PP}/${PPS}; \
+			mk-build-deps -i -r --host-arch ${DISTRO_ARCH} -t \"apt-get -y --no-install-recommends --allow-downgrades -o Debug::pkgProblemResolver=yes\" debian/control; \
+		"
+
+		# Create kernel configuration for kbuild
+		dpkg_configure_kernel ${KERNEL_KBUILD_DIR}-target
+		KERNEL_CONFIG_TARGET="${@get_kernel_config_target(d)}"
+
+		schroot -r -c $schroot_session -- sh -c " \
+			cd ${PP}/${PPS}; \
+			ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${KERNEL_CONFIG_TARGET}; \
+		"
+
+		cd ${S}
+		ARCH=${KERNEL_ARCH} ./scripts/kconfig/merge_config.sh -m -O ${KERNEL_KBUILD_DIR}-target/ ${KERNEL_KBUILD_DIR}-target/.config ${KERNEL_CONFIG_FRAGMENTS}
+		cd -
+
+	        # Build kernel scripts and tools for target arch in a non-cross way
+	        # Cross building kernel scripts and tools would not provide them in target architecture
+		schroot -r -c $schroot_session -- sh -c " \
+			cd ${PP}/${PPS}; \
+			ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target olddefconfig; \
+			ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${PARALLEL_MAKE} scripts; \
+			if [ -d "tools/objtool" ]; then \
+				echo ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${PARALLEL_MAKE} tools/objtool; \
+				ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${PARALLEL_MAKE} tools/objtool; \
+			fi; \
+			if grep -q "CONFIG_MODULES=y" ${KERNEL_KBUILD_DIR}-target/.config; then \
+				echo ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${PARALLEL_MAKE} modules_prepare; \
+				ARCH=${KERNEL_ARCH} make O=${KERNEL_KBUILD_DIR}-target ${PARALLEL_MAKE} modules_prepare; \
+			fi; \
+		"
+
+		# End kernel schroot-target session
+		schroot -e -c $schroot_session
+
+		# Restore kernel schroot-host configuration
+		schroot_create_configs
+	fi
 }
-- 
2.30.2