[RFC 2/5] meta: Implement two stage build

[Anton Mikanovich <amikan@ilbers.de>]

Split package build into two stages: building source package and
building binary package. This allows to obtain Debian package
dependencies before any build steps will actually start.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>
---
 meta/classes/dpkg-base.bbclass                | 34 +++++++++++++++++--
 meta/classes/dpkg-gbp.bbclass                 |  6 +---
 meta/classes/dpkg-prebuilt.bbclass            |  7 ++++
 meta/classes/dpkg.bbclass                     | 21 +++++++++---
 meta/classes/rootfs.bbclass                   |  2 ++
 .../buildchroot/buildchroot.inc               |  2 ++
 .../buildchroot/files/build-source.sh         | 16 +++++++++
 .../buildchroot/files/build.sh                |  2 +-
 8 files changed, 76 insertions(+), 14 deletions(-)
 create mode 100644 meta/recipes-devtools/buildchroot/files/build-source.sh

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index e3ec133..916bae7 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -75,7 +75,7 @@ addtask adjust_git after do_unpack before do_patch
 do_adjust_git[lockfiles] += "${DL_DIR}/git/isar.lock"
 
 inherit patch
-addtask patch after do_adjust_git before do_dpkg_build
+addtask patch after do_adjust_git before do_dpkg_source
 
 SRC_APT ?= ""
 
@@ -175,7 +175,7 @@ do_prepare_build() {
     true
 }
 
-addtask prepare_build after do_patch do_transform_template before do_dpkg_build
+addtask prepare_build after do_patch do_transform_template before do_dpkg_source
 # If Isar recipes depend on each other, they typically need the package
 # deployed to isar-apt
 do_prepare_build[deptask] = "do_deploy_deb"
@@ -238,6 +238,19 @@ python do_dpkg_build() {
 
 addtask dpkg_build
 
+python do_dpkg_source() {
+    lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock",
+                             shared=True)
+    bb.build.exec_func("dpkg_do_mounts", d)
+    try:
+        bb.build.exec_func("dpkg_runbuild_source", d)
+    finally:
+        bb.build.exec_func("dpkg_undo_mounts", d)
+        bb.utils.unlockfile(lock)
+}
+
+addtask dpkg_source before do_dpkg_build
+
 SSTATETASKS += "do_dpkg_build"
 DPKG_SSTATE = "${WORKDIR}/dpkg-sstate"
 do_dpkg_build[dirs] += "${DPKG_SSTATE} ${S}/.."
@@ -300,6 +313,21 @@ do_deploy_deb[depends] += "isar-apt:do_cache_config"
 do_deploy_deb[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
 do_deploy_deb[dirs] = "${S}"
 
+do_deploy_source() {
+    #TODO: dsc_clean
+    find "${S}/../" -name '*\.dsc' | while read package; do
+        repo_add_srcpackage "${REPO_ISAR_DIR}"/"${DISTRO}" \
+            "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+            "${DEBDISTRONAME}" \
+            "${package}"
+    done
+}
+
+addtask deploy_source after do_dpkg_source before do_dpkg_build
+do_deploy_source[depends] += "isar-apt:do_cache_config"
+do_deploy_source[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
+do_deploy_source[dirs] = "${S}"
+
 python do_devshell() {
     bb.build.exec_func('dpkg_do_mounts', d)
 
@@ -327,6 +355,6 @@ python do_devshell_nodeps() {
 
 # devshell may be placed after do_instell_builddeps in downstream classes.
 # devshell_nodeps will always stay right after do_prepare_build.
-addtask devshell_nodeps after do_prepare_build
+addtask devshell_nodeps after do_deploy_source
 do_devshell_nodeps[dirs] = "${DEVSHELL_STARTDIR}"
 do_devshell_nodeps[nostamp] = "1"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index d956e8c..b02aed0 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,8 +12,7 @@ PATCHTOOL ?= "git"
 GBP_DEPENDS ?= "git-buildpackage pristine-tar"
 GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
 
-do_install_builddeps_append() {
-    dpkg_do_mounts
+dpkg_runbuild_source_prepend() {
     distro="${DISTRO}"
     if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
        distro="${HOST_DISTRO}"
@@ -26,9 +25,6 @@ do_install_builddeps_append() {
     sudo -E chroot ${BUILDCHROOT_DIR} \
         apt-get install -y -o Debug::pkgProblemResolver=yes \
                         --no-install-recommends ${GBP_DEPENDS}
-    dpkg_undo_mounts
-}
 
-dpkg_runbuild_prepend() {
     export GBP_PREFIX="gbp buildpackage --git-ignore-new ${GBP_EXTRA_OPTIONS} --git-builder="
 }
diff --git a/meta/classes/dpkg-prebuilt.bbclass b/meta/classes/dpkg-prebuilt.bbclass
index c32224b..c5d645c 100644
--- a/meta/classes/dpkg-prebuilt.bbclass
+++ b/meta/classes/dpkg-prebuilt.bbclass
@@ -18,3 +18,10 @@ python do_unpack_prepend() {
 
 deltask dpkg_build
 addtask unpack before do_deploy_deb
+
+deltask deploy_source
+# dummy task for 1st stage
+do_deploy_source() {
+    true
+}
+addtask deploy_source after do_deploy_deb before do_build
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 320102b..e95420a 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -17,15 +17,17 @@ do_install_builddeps() {
     fi
 
     deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}"
-    sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
-        ${PP}/${PPS} ${PACKAGE_ARCH} --download-only
+    sudo -E chroot ${BUILDCHROOT_DIR} sh -c "apt-get update && apt-get \
+        -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+        --allow-downgrades --download-only build-dep -a ${PACKAGE_ARCH} ${PN}"
     deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}"
-    sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
-        ${PP}/${PPS} ${PACKAGE_ARCH}
+    sudo -E chroot ${BUILDCHROOT_DIR} sh -c "apt-get update && apt-get \
+        -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+        --allow-downgrades build-dep -a ${PACKAGE_ARCH} ${PN}"
     dpkg_undo_mounts
 }
 
-addtask install_builddeps after do_prepare_build before do_dpkg_build
+addtask install_builddeps after do_deploy_source before do_dpkg_build
 do_install_builddeps[depends] += "isar-apt:do_cache_config"
 # apt and reprepro may not run in parallel, acquire the Isar lock
 do_install_builddeps[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
@@ -42,3 +44,12 @@ dpkg_runbuild() {
     sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
          /isar/build.sh ${PP}/${PPS} ${PACKAGE_ARCH}
 }
+
+dpkg_runbuild_source() {
+    E="${@ isar_export_proxies(d)}"
+    export DEB_BUILD_OPTIONS="${@ isar_deb_build_options(d)}"
+    export DEB_BUILD_PROFILES="${@ isar_deb_build_profiles(d)}"
+    export PARALLEL_MAKE="${PARALLEL_MAKE}"
+    sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+         /isar/build-source.sh ${PP}/${PPS} ${PACKAGE_ARCH}
+}
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index d578dd3..c5ea748 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -88,6 +88,8 @@ rootfs_configure_isar_apt() {
     mkdir -p '${ROOTFSDIR}/etc/apt/sources.list.d'
     echo 'deb [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' > \
         '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
+    echo 'deb-src [trusted=yes] file:///isar-apt ${DEBDISTRONAME} main' >> \
+        '${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list'
 
     mkdir -p '${ROOTFSDIR}/etc/apt/preferences.d'
     cat << EOF > '${ROOTFSDIR}/etc/apt/preferences.d/isar-apt'
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
index aa190e9..2872680 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.inc
+++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
@@ -9,6 +9,7 @@ LIC_FILES_CHKSUM = "file://${LAYERDIR_core}/licenses/COPYING.GPLv2;md5=751419260
 FILESPATH_prepend := "${THISDIR}/files:"
 SRC_URI = "file://configscript.sh \
            file://build.sh \
+           file://build-source.sh \
            file://common.sh \
            file://deps.sh"
 PV = "1.0"
@@ -58,6 +59,7 @@ buildchroot_install_files() {
     sudo chmod -R a+rw "${BUILDCHROOT_DIR}/home/builder"
     sudo install -m 755 -d ${BUILDCHROOT_DIR}/isar
     sudo install -m 755 ${WORKDIR}/build.sh ${BUILDCHROOT_DIR}/isar/
+    sudo install -m 755 ${WORKDIR}/build-source.sh ${BUILDCHROOT_DIR}/isar/
     sudo install -m 755 ${WORKDIR}/common.sh ${BUILDCHROOT_DIR}/isar/
     sudo install -m 755 ${WORKDIR}/deps.sh ${BUILDCHROOT_DIR}/isar/
 
diff --git a/meta/recipes-devtools/buildchroot/files/build-source.sh b/meta/recipes-devtools/buildchroot/files/build-source.sh
new file mode 100644
index 0000000..a327c9b
--- /dev/null
+++ b/meta/recipes-devtools/buildchroot/files/build-source.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This software is a part of ISAR.
+# Copyright (C) 2022 ilbers GmbH
+
+source /isar/common.sh
+
+# If autotools files have been created, update their timestamp to
+# prevent them from being regenerated
+for i in configure aclocal.m4 Makefile.am Makefile.in; do
+    if [ -f "${i}" ]; then
+        touch "${i}"
+    fi
+done
+
+${GBP_PREFIX}dpkg-buildpackage -a$target_arch -d --source-option=-I -S
diff --git a/meta/recipes-devtools/buildchroot/files/build.sh b/meta/recipes-devtools/buildchroot/files/build.sh
index d4709cf..c19221b 100644
--- a/meta/recipes-devtools/buildchroot/files/build.sh
+++ b/meta/recipes-devtools/buildchroot/files/build.sh
@@ -14,4 +14,4 @@ for i in configure aclocal.m4 Makefile.am Makefile.in; do
     fi
 done
 
-${GBP_PREFIX}dpkg-buildpackage -a$target_arch --source-option=-I
+${GBP_PREFIX}dpkg-buildpackage -a$target_arch --source-option=-I -b
-- 
2.25.1