[RFC v2][PATCH 0/3] Introduce base-apt

[RFC v2][PATCH 0/3] Introduce base-apt

[Alexander Smirnov]

Hello everybody,

first of all, to avoid unnecessary effort I've diced to split build
reproducibility feature into two parts:
 - base-apt creation
 - switch Isar to base-apt

So this series introduces base-apt only.

If set ISAR_BUILD_REP = "1", Isar creates base-apt repository that contains
debs for all possible build targets within current tree.

Special notes:
 - To derive dependencies, I've added DEBIAN_DEPENDS variable which duplicates
   'debian/control' content. IMHO this is the best compromise for now.
 - I use multistrap to fetch debs, I haven't found for now any other tool that
   could fetch debs from multiple sources. The issue is that multistrap doesn't
   understand 'debian/control' format, so perl parsing is used.

Alexander Smirnov (3):
  dpkg-base: Make DEBIAN_DEPENDS global
  build-rep: Add helper class
  base-apt: Introduce fetching upstream apt

 meta-isar/conf/local.conf.sample                   |  6 ++
 meta-isar/recipes-app/hello/hello.bb               |  2 +
 meta/classes/build-rep.bbclass                     | 32 ++++++++
 meta/classes/dpkg-base.bbclass                     |  4 +
 meta/classes/dpkg-raw.bbclass                      |  1 -
 meta/classes/image.bbclass                         |  2 +
 meta/conf/isar-bitbake.conf                        |  2 +
 meta/recipes-devtools/base-apt/base-apt.bb         | 95 ++++++++++++++++++++++
 .../base-apt/files/distributions.in                |  3 +
 .../base-apt/files/multistrap.conf.in              | 28 +++++++
 meta/recipes-devtools/buildchroot/buildchroot.bb   | 10 +++
 11 files changed, 184 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/build-rep.bbclass
 create mode 100644 meta/recipes-devtools/base-apt/base-apt.bb
 create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
 create mode 100644 meta/recipes-devtools/base-apt/files/multistrap.conf.in

-- 
2.1.4

[RFC v2][PATCH 1/3] dpkg-base: Make DEBIAN_DEPENDS global

[Alexander Smirnov]

Make DEBIAN_DEPENDS variable global for dpkg packages being built from
source code.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
---
 meta/classes/dpkg-base.bbclass | 2 ++
 meta/classes/dpkg-raw.bbclass  | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 4941f9b..4d220da 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -1,6 +1,8 @@
 # This software is a part of ISAR.
 # Copyright (C) 2017 Siemens AG
 
+DEBIAN_DEPENDS ?= ""
+
 # Add dependency from buildchroot creation
 do_build[depends] = "buildchroot:do_build"
 
diff --git a/meta/classes/dpkg-raw.bbclass b/meta/classes/dpkg-raw.bbclass
index d132066..3d41608 100644
--- a/meta/classes/dpkg-raw.bbclass
+++ b/meta/classes/dpkg-raw.bbclass
@@ -3,7 +3,6 @@
 
 inherit dpkg-base
 
-DEBIAN_DEPENDS ?= ""
 MAINTAINER ?= "FIXME Unknown maintainer"
 
 D = "${WORKDIR}/image/"
-- 
2.1.4

[RFC v2][PATCH 2/3] build-rep: Add helper class

[Alexander Smirnov]

Add class that helps to implement build reproducibility. It implements
anonymous function that will get all the Debian dependencies that are
needed for current Isar tree.

Until build reproducibility will be fully implemented, it's disabled by
default. To enable it just set ISAR_BUILD_REP to "1" in local.conf.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
---
 meta-isar/conf/local.conf.sample                 |  6 +++++
 meta-isar/recipes-app/hello/hello.bb             |  2 ++
 meta/classes/build-rep.bbclass                   | 32 ++++++++++++++++++++++++
 meta/classes/dpkg-base.bbclass                   |  2 ++
 meta/classes/image.bbclass                       |  2 ++
 meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++
 6 files changed, 46 insertions(+)
 create mode 100644 meta/classes/build-rep.bbclass

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 660958f..45b8995 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
 #
 # Number of attempts to try to get reprepro lock for access to apt cache
 REPREPRO_LOCK_ATTEMPTS = "16"
+
+# Isar build reproducibility feature creates local repository which contains
+# copies for all the upstream Debian packages that could be used to build
+# your Isar tree. So fetching them once will guarantee that all the next Isar
+# builds will be identically.
+ISAR_BUILD_REP ?= "0"
diff --git a/meta-isar/recipes-app/hello/hello.bb b/meta-isar/recipes-app/hello/hello.bb
index 44b8bc3..fafda2e 100644
--- a/meta-isar/recipes-app/hello/hello.bb
+++ b/meta-isar/recipes-app/hello/hello.bb
@@ -16,3 +16,5 @@ SRCREV = "ad7065ecc4840cc436bfcdac427386dbba4ea719"
 SRC_DIR = "git"
 
 inherit dpkg
+
+DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
diff --git a/meta/classes/build-rep.bbclass b/meta/classes/build-rep.bbclass
new file mode 100644
index 0000000..ede5a93
--- /dev/null
+++ b/meta/classes/build-rep.bbclass
@@ -0,0 +1,32 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017 Siemens AG
+
+python __anonymous() {
+    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
+    if rep == "0":
+        return
+
+    depsdir = d.getVar('BASE_APT_DIR', True)
+    if depsdir is None:
+        return
+
+    depsdir += '/deps/'
+
+    pn = d.getVar('PN', True)
+
+    if not os.path.exists(depsdir):
+        os.makedirs(depsdir, exist_ok=True)
+
+    if d.getVar('DEBIAN_DEPENDS', True):
+        with open(depsdir + '/' +  pn + '.depends', 'w') as the_file:
+            the_file.write(d.getVar('DEBIAN_DEPENDS', True))
+            the_file.close
+    elif d.getVar('BUILDCHROOT_PREINSTALL', True):
+        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
+            the_file.write(d.getVar('BUILDCHROOT_PREINSTALL', True))
+            the_file.close
+    elif d.getVar('IMAGE_PREINSTALL', True):
+        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
+            the_file.write(d.getVar('IMAGE_PREINSTALL', True))
+            the_file.close
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 4d220da..bf66e78 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -3,6 +3,8 @@
 
 DEBIAN_DEPENDS ?= ""
 
+inherit build-rep
+
 # Add dependency from buildchroot creation
 do_build[depends] = "buildchroot:do_build"
 
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index e2cb01b..67f5af8 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -5,6 +5,8 @@ IMAGE_INSTALL ?= ""
 IMAGE_TYPE    ?= "ext4-img"
 IMAGE_ROOTFS   = "${WORKDIR}/rootfs"
 
+inherit build-rep
+
 def get_image_name(d, name_link):
     S = d.getVar("IMAGE_ROOTFS", True)
     path_link = os.path.join(S, name_link)
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index 51f9d5d..da18231 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -16,6 +16,8 @@ SRC_URI = "file://multistrap.conf.in \
            file://build.sh"
 PV = "1.0"
 
+inherit build-rep
+
 BUILDCHROOT_PREINSTALL ?= "gcc \
                            make \
                            build-essential \
-- 
2.1.4

[RFC v2][PATCH 3/3] base-apt: Introduce fetching upstream apt

[Alexander Smirnov]

This patch introduces mechanism how to fetch deb packages from dedicated
upstream apt repos and store them localy. Local repository is called
'base-apt' and it will be used to generate buildchroot and image root
filesystems. Using 'base-apt' will guarantee build reproducibility between
builds.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
---
 meta/conf/isar-bitbake.conf                        |  2 +
 meta/recipes-devtools/base-apt/base-apt.bb         | 95 ++++++++++++++++++++++
 .../base-apt/files/distributions.in                |  3 +
 .../base-apt/files/multistrap.conf.in              | 28 +++++++
 meta/recipes-devtools/buildchroot/buildchroot.bb   |  8 ++
 5 files changed, 136 insertions(+)
 create mode 100644 meta/recipes-devtools/base-apt/base-apt.bb
 create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
 create mode 100644 meta/recipes-devtools/base-apt/files/multistrap.conf.in

diff --git a/meta/conf/isar-bitbake.conf b/meta/conf/isar-bitbake.conf
index 5a26743..df54399 100644
--- a/meta/conf/isar-bitbake.conf
+++ b/meta/conf/isar-bitbake.conf
@@ -23,6 +23,8 @@ DEPLOY_DIR_DEB = "${TMPDIR}/deploy/deb/${MACHINE}"
 SSTATE_DIR ?= "${TMPDIR}/sstate-cache"
 BUILDCHROOT_DIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/buildchroot/rootfs"
 
+BASE_APT_DIR ?= "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/base-apt"
+
 # Setup our default hash policy
 BB_SIGNATURE_HANDLER ?= "noop"
 
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
new file mode 100644
index 0000000..e05ea61
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -0,0 +1,95 @@
+# Caching upstream apt repository to local one.
+#
+# This software is a part of ISAR.
+# Copyright (C) 2015-2017 ilbers GmbH
+
+DESCRIPTION = "Upstream apt caching"
+
+LICENSE = "gpl-2.0"
+LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
+
+FILESPATH =. "${LAYERDIR_core}/recipes-devtools/base-apt/files:"
+SRC_URI = "file://distributions.in \
+           file://multistrap.conf.in \
+          "
+
+BASE_PREINSTALL ?= ""
+
+WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+
+do_fetch_debs[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+
+do_fetch_debs() {
+    for package in `ls ${WORKDIR}/deps/*.depends`; do
+        DEPS=$(perl -ne 'next if /^#/; $p=(s/,|\n|\([^)]+\)|\[[^]]+\]//mg); print if $p' < $package)
+        PACKAGES="$PACKAGES $DEPS"
+    done
+
+    for package in `ls ${WORKDIR}/deps/*.preinst`; do
+        DEPS=$(cat $package | xargs)
+        PACKAGES="$PACKAGES $DEPS"
+    done
+
+    PACKAGES="$PACKAGES $BASE_PREINSTALL"
+
+    # Adjust multistrap config
+    sed -e 's|##BASE_PREINSTALL##|'"$PACKAGES"'|g' \
+        -e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
+        -e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
+        -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
+        -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
+        -e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
+        -e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
+        -e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/configscript.sh|g' \
+        -e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
+        -e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
+           "${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
+
+    # Fetch deb packages
+    sudo -E multistrap \
+        -a ${DISTRO_ARCH} \
+        -d "${WORKDIR}/download" \
+        -f "${WORKDIR}/multistrap.conf" \
+        > ${WORKDIR}/multistrap.log 2>&1
+}
+
+addtask fetch_debs after do_unpack before do_build
+
+BASE_APT_CONF_DIR = "${BASE_APT_DIR}/apt/conf"
+do_build[dirs] = "${BASE_APT_CONF_DIR}"
+do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+
+do_build() {
+    if [ ! -e "${BASE_APT_CONF_DIR}/distributions" ]; then
+        sed -e "s#{DISTRO_NAME}#"${DISTRO_SUITE}"#g" \
+            ${WORKDIR}/distributions.in > ${BASE_APT_CONF_DIR}/distributions
+    fi
+
+    # Create reprepro cache
+    if [ ! -d "${BASE_APT_DIR}/apt" ]; then
+        reprepro -b ${BASE_APT_DIR}/apt \
+                 --dbdir ${BASE_APT_DIR}/db \
+                 export ${DISTRO_SUITE}
+    fi
+
+    # Process all the packages fetched by multistrap
+    for deb in $(ls ${WORKDIR}/download/var/cache/apt/archives/*.deb);
+    do
+        pn=$(dpkg-deb -I $deb | grep 'Package:' | cut -d ' ' -f 3)
+        pv=$(dpkg-deb -I $deb | grep 'Version:' | cut -d ' ' -f 3)
+        line=$(cat ${WORKDIR}/multistrap.log | grep -E '^(Get:)' | grep " $pn ")
+        url=$(echo $line | cut -d ' ' -f 2)
+        component=$(basename $(echo $line | cut -d ' ' -f 3))
+
+        # Store download history
+        echo $pn $pv $component $url >> ${WORKDIR}/deb.list
+
+        reprepro -b ${BASE_APT_DIR}/apt \
+                 --dbdir ${BASE_APT_DIR}/db \
+                 -C $component \
+                 includedeb ${DISTRO_SUITE} \
+                 $deb
+    done
+
+    sudo rm -rf ${WORKDIR}/download
+}
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
new file mode 100644
index 0000000..44e9513
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/files/distributions.in
@@ -0,0 +1,3 @@
+Codename: {DISTRO_NAME}
+Architectures: i386 armhf amd64 source
+Components: main contrib non-free firmware
diff --git a/meta/recipes-devtools/base-apt/files/multistrap.conf.in b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
new file mode 100644
index 0000000..27bf985
--- /dev/null
+++ b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
@@ -0,0 +1,28 @@
+# This software is a part of ISAR.
+# Copyright (C) 2015-2017 ilbers GmbH
+
+[General]
+noauth=true
+unpack=false
+ignorenativearch=true
+bootstrap=##DISTRO_MULTICONF_BOOTSTRAP##
+aptsources=##DISTRO_MULTICONF_APTSOURCES##
+
+[base]
+source=##DISTRO_APT_SOURCE##
+suite=##DISTRO_SUITE##
+components=##DISTRO_COMPONENTS##
+packages=##BASE_PREINSTALL##
+omitdebsrc=true
+
+[updates]
+source=##DISTRO_APT_SOURCE##
+suite=##DISTRO_SUITE##-updates
+components=##DISTRO_COMPONENTS##
+omitdebsrc=true
+
+[security]
+source=##DISTRO_APT_SOURCE_SEC##
+suite=##DISTRO_SUITE##/updates
+components=##DISTRO_COMPONENTS##
+omitdebsrc=true
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index da18231..a551e50 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -33,6 +33,14 @@ BUILDCHROOT_PREINSTALL ?= "gcc \
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
 
+python __anonymous() {
+    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
+    if rep == "0":
+        return
+
+    d.setVarFlag("do_build", "depends", "base-apt:do_build")
+}
+
 do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_build[dirs] = "${WORKDIR}/hooks_multistrap"
 
-- 
2.1.4

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Jan Kiszka]

On 2018-01-11 12:19, Alexander Smirnov wrote:
> Add class that helps to implement build reproducibility. It implements
> anonymous function that will get all the Debian dependencies that are
> needed for current Isar tree.
> 
> Until build reproducibility will be fully implemented, it's disabled by
> default. To enable it just set ISAR_BUILD_REP to "1" in local.conf.

"ISAR_REPRODUCIBLE_BUILD" - "REP" could mean anything.

Still feels like this should rather default to "on", so maybe call it
"ISAR_NON_REPRODUCIBLE_BUILD".

Jan

> 
> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> ---
>  meta-isar/conf/local.conf.sample                 |  6 +++++
>  meta-isar/recipes-app/hello/hello.bb             |  2 ++
>  meta/classes/build-rep.bbclass                   | 32 ++++++++++++++++++++++++
>  meta/classes/dpkg-base.bbclass                   |  2 ++
>  meta/classes/image.bbclass                       |  2 ++
>  meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++
>  6 files changed, 46 insertions(+)
>  create mode 100644 meta/classes/build-rep.bbclass
> 
> diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
> index 660958f..45b8995 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>  #
>  # Number of attempts to try to get reprepro lock for access to apt cache
>  REPREPRO_LOCK_ATTEMPTS = "16"
> +
> +# Isar build reproducibility feature creates local repository which contains
> +# copies for all the upstream Debian packages that could be used to build
> +# your Isar tree. So fetching them once will guarantee that all the next Isar
> +# builds will be identically.
> +ISAR_BUILD_REP ?= "0"
> diff --git a/meta-isar/recipes-app/hello/hello.bb b/meta-isar/recipes-app/hello/hello.bb
> index 44b8bc3..fafda2e 100644
> --- a/meta-isar/recipes-app/hello/hello.bb
> +++ b/meta-isar/recipes-app/hello/hello.bb
> @@ -16,3 +16,5 @@ SRCREV = "ad7065ecc4840cc436bfcdac427386dbba4ea719"
>  SRC_DIR = "git"
>  
>  inherit dpkg
> +
> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
> diff --git a/meta/classes/build-rep.bbclass b/meta/classes/build-rep.bbclass
> new file mode 100644
> index 0000000..ede5a93
> --- /dev/null
> +++ b/meta/classes/build-rep.bbclass
> @@ -0,0 +1,32 @@
> +# This software is a part of ISAR.
> +# Copyright (C) 2017 Siemens AG
> +
> +python __anonymous() {
> +    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
> +    if rep == "0":
> +        return
> +
> +    depsdir = d.getVar('BASE_APT_DIR', True)
> +    if depsdir is None:
> +        return
> +
> +    depsdir += '/deps/'
> +
> +    pn = d.getVar('PN', True)
> +
> +    if not os.path.exists(depsdir):
> +        os.makedirs(depsdir, exist_ok=True)
> +
> +    if d.getVar('DEBIAN_DEPENDS', True):
> +        with open(depsdir + '/' +  pn + '.depends', 'w') as the_file:
> +            the_file.write(d.getVar('DEBIAN_DEPENDS', True))
> +            the_file.close
> +    elif d.getVar('BUILDCHROOT_PREINSTALL', True):
> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
> +            the_file.write(d.getVar('BUILDCHROOT_PREINSTALL', True))
> +            the_file.close
> +    elif d.getVar('IMAGE_PREINSTALL', True):
> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
> +            the_file.write(d.getVar('IMAGE_PREINSTALL', True))
> +            the_file.close
> +}
> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
> index 4d220da..bf66e78 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -3,6 +3,8 @@
>  
>  DEBIAN_DEPENDS ?= ""
>  
> +inherit build-rep
> +
>  # Add dependency from buildchroot creation
>  do_build[depends] = "buildchroot:do_build"
>  
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index e2cb01b..67f5af8 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -5,6 +5,8 @@ IMAGE_INSTALL ?= ""
>  IMAGE_TYPE    ?= "ext4-img"
>  IMAGE_ROOTFS   = "${WORKDIR}/rootfs"
>  
> +inherit build-rep
> +
>  def get_image_name(d, name_link):
>      S = d.getVar("IMAGE_ROOTFS", True)
>      path_link = os.path.join(S, name_link)
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
> index 51f9d5d..da18231 100644
> --- a/meta/recipes-devtools/buildchroot/buildchroot.bb
> +++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
> @@ -16,6 +16,8 @@ SRC_URI = "file://multistrap.conf.in \
>             file://build.sh"
>  PV = "1.0"
>  
> +inherit build-rep
> +
>  BUILDCHROOT_PREINSTALL ?= "gcc \
>                             make \
>                             build-essential \
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Henning Schild]

Am Thu, 11 Jan 2018 14:19:38 +0300
schrieb Alexander Smirnov <asmirnov@ilbers.de>:

> Add class that helps to implement build reproducibility. It implements
> anonymous function that will get all the Debian dependencies that are
> needed for current Isar tree.
> 
> Until build reproducibility will be fully implemented, it's disabled
> by default. To enable it just set ISAR_BUILD_REP to "1" in local.conf.
> 
> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> ---
>  meta-isar/conf/local.conf.sample                 |  6 +++++
>  meta-isar/recipes-app/hello/hello.bb             |  2 ++
>  meta/classes/build-rep.bbclass                   | 32
> ++++++++++++++++++++++++
> meta/classes/dpkg-base.bbclass                   |  2 ++
> meta/classes/image.bbclass                       |  2 ++
> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
> changed, 46 insertions(+) create mode 100644
> meta/classes/build-rep.bbclass
> 
> diff --git a/meta-isar/conf/local.conf.sample
> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>  #
>  # Number of attempts to try to get reprepro lock for access to apt
> cache REPREPRO_LOCK_ATTEMPTS = "16"
> +
> +# Isar build reproducibility feature creates local repository which
> contains +# copies for all the upstream Debian packages that could be
> used to build +# your Isar tree. So fetching them once will guarantee
> that all the next Isar +# builds will be identically.
> +ISAR_BUILD_REP ?= "0"
> diff --git a/meta-isar/recipes-app/hello/hello.bb
> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e 100644
> --- a/meta-isar/recipes-app/hello/hello.bb
> +++ b/meta-isar/recipes-app/hello/hello.bb
> @@ -16,3 +16,5 @@ SRCREV = "ad7065ecc4840cc436bfcdac427386dbba4ea719"
>  SRC_DIR = "git"
>  
>  inherit dpkg
> +
> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"

If i understand it correctly, this approach is an absolute NoGo.

DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
packages. Here you include build-deps of a package that Isar needs to
build. And you do so by introducing a copy of a string that is already
included in the sources /debian/-folder. In fact you would have to
put build and runtime deps into that one variable. And what about
packages where the content depends on DISTRO_ARCH and friends?

I think we first need a working solution of two "inherit dpkg" packages
actually depending on each-other. After that we will need a real build
of the Image to fill the cache. That is the only reliable way to make
sure that the repo will contain the runtime and the build deps of all
packages. More "sed"-guesswork is not going to do that trick and is
only getting us 95% of what we need. And maintaining copies of stuff
that is in /debian/ should not be considered.

Henning

> diff --git a/meta/classes/build-rep.bbclass
> b/meta/classes/build-rep.bbclass new file mode 100644
> index 0000000..ede5a93
> --- /dev/null
> +++ b/meta/classes/build-rep.bbclass
> @@ -0,0 +1,32 @@
> +# This software is a part of ISAR.
> +# Copyright (C) 2017 Siemens AG
> +
> +python __anonymous() {
> +    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
> +    if rep == "0":
> +        return
> +
> +    depsdir = d.getVar('BASE_APT_DIR', True)
> +    if depsdir is None:
> +        return
> +
> +    depsdir += '/deps/'
> +
> +    pn = d.getVar('PN', True)
> +
> +    if not os.path.exists(depsdir):
> +        os.makedirs(depsdir, exist_ok=True)
> +
> +    if d.getVar('DEBIAN_DEPENDS', True):
> +        with open(depsdir + '/' +  pn + '.depends', 'w') as the_file:
> +            the_file.write(d.getVar('DEBIAN_DEPENDS', True))
> +            the_file.close
> +    elif d.getVar('BUILDCHROOT_PREINSTALL', True):
> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
> +            the_file.write(d.getVar('BUILDCHROOT_PREINSTALL', True))
> +            the_file.close
> +    elif d.getVar('IMAGE_PREINSTALL', True):
> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
> +            the_file.write(d.getVar('IMAGE_PREINSTALL', True))
> +            the_file.close
> +}
> diff --git a/meta/classes/dpkg-base.bbclass
> b/meta/classes/dpkg-base.bbclass index 4d220da..bf66e78 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -3,6 +3,8 @@
>  
>  DEBIAN_DEPENDS ?= ""
>  
> +inherit build-rep
> +
>  # Add dependency from buildchroot creation
>  do_build[depends] = "buildchroot:do_build"
>  
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index e2cb01b..67f5af8 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -5,6 +5,8 @@ IMAGE_INSTALL ?= ""
>  IMAGE_TYPE    ?= "ext4-img"
>  IMAGE_ROOTFS   = "${WORKDIR}/rootfs"
>  
> +inherit build-rep
> +
>  def get_image_name(d, name_link):
>      S = d.getVar("IMAGE_ROOTFS", True)
>      path_link = os.path.join(S, name_link)
> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb
> b/meta/recipes-devtools/buildchroot/buildchroot.bb index
> 51f9d5d..da18231 100644 ---
> a/meta/recipes-devtools/buildchroot/buildchroot.bb +++
> b/meta/recipes-devtools/buildchroot/buildchroot.bb @@ -16,6 +16,8 @@
> SRC_URI = "file://multistrap.conf.in \ file://build.sh"
>  PV = "1.0"
>  
> +inherit build-rep
> +
>  BUILDCHROOT_PREINSTALL ?= "gcc \
>                             make \
>                             build-essential \

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Alexander Smirnov]

On 01/12/2018 03:32 PM, Henning Schild wrote:
> Am Thu, 11 Jan 2018 14:19:38 +0300
> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
> 
>> Add class that helps to implement build reproducibility. It implements
>> anonymous function that will get all the Debian dependencies that are
>> needed for current Isar tree.
>>
>> Until build reproducibility will be fully implemented, it's disabled
>> by default. To enable it just set ISAR_BUILD_REP to "1" in local.conf.
>>
>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>> ---
>>   meta-isar/conf/local.conf.sample                 |  6 +++++
>>   meta-isar/recipes-app/hello/hello.bb             |  2 ++
>>   meta/classes/build-rep.bbclass                   | 32
>> ++++++++++++++++++++++++
>> meta/classes/dpkg-base.bbclass                   |  2 ++
>> meta/classes/image.bbclass                       |  2 ++
>> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
>> changed, 46 insertions(+) create mode 100644
>> meta/classes/build-rep.bbclass
>>
>> diff --git a/meta-isar/conf/local.conf.sample
>> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
>> --- a/meta-isar/conf/local.conf.sample
>> +++ b/meta-isar/conf/local.conf.sample
>> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>>   #
>>   # Number of attempts to try to get reprepro lock for access to apt
>> cache REPREPRO_LOCK_ATTEMPTS = "16"
>> +
>> +# Isar build reproducibility feature creates local repository which
>> contains +# copies for all the upstream Debian packages that could be
>> used to build +# your Isar tree. So fetching them once will guarantee
>> that all the next Isar +# builds will be identically.
>> +ISAR_BUILD_REP ?= "0"
>> diff --git a/meta-isar/recipes-app/hello/hello.bb
>> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e 100644
>> --- a/meta-isar/recipes-app/hello/hello.bb
>> +++ b/meta-isar/recipes-app/hello/hello.bb
>> @@ -16,3 +16,5 @@ SRCREV = "ad7065ecc4840cc436bfcdac427386dbba4ea719"
>>   SRC_DIR = "git"
>>   
>>   inherit dpkg
>> +
>> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
> 
> If i understand it correctly, this approach is an absolute NoGo.
> 
> DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
> packages. Here you include build-deps of a package that Isar needs to
> build. And you do so by introducing a copy of a string that is already
> included in the sources /debian/-folder. In fact you would have to
> put build and runtime deps into that one variable. And what about
> packages where the content depends on DISTRO_ARCH and friends?
> 
> I think we first need a working solution of two "inherit dpkg" packages
> actually depending on each-other. After that we will need a real build
> of the Image to fill the cache. That is the only reliable way to make
> sure that the repo will contain the runtime and the build deps of all
> packages. More "sed"-guesswork is not going to do that trick and is
> only getting us 95% of what we need. And maintaining copies of stuff
> that is in /debian/ should not be considered.

Let's look at this realistically:

1. The upstream Debian should be fetched via single operation. If you 
want to populate the cache during image build - you can't guarantee that 
all the packages were built in the same environment. As Christoph wrote, 
at any time the upstream apt could be updated. So the buildchroot and 
image filesystems should be generated using local static apt, otherwise 
it makes no sense to claim reproducibility.

2. Regarding 'debian/control' duplications. According to the idea, that 
'base-apt' should be generated first of all, in bitbake terms it means that:

  *.bb:do_build() should depends on base-apt:do_build().

On the other hand, base-apt:do_build() should already know the full list 
of all deps for *.bb. So for now I see two ways how to implement this:

  - Duplicate 'debian/control' in recipe and share this variable with 
base-apt (like it's done in this series).

  - Manually parse 'debian/control'. But in this case base-apt should 
depend on *.bb:do_unpack. Due to bitbake is 'statically configured' 
tool, base-apt should know the list of all the recipes, what is also 
impossible. In previous series I've tried to implement this by defining 
a list of images in some global variable.

2. Question how to get the list of build and runtime dependencies is 
still open, I've already asked about this in previous mail. So, the goal 
is to have the full list of packages that should be fetched. 
'debian/control' contains very specific format, so I can't feed this 
line to multistrap/apt-get as it is. I see the following solutions:
  - Implement custom tool to do this (apt/dpkg uses standard perl libs 
which contains all this stuff).
  - Manually parse 'debian/control'
  - Drop the requirement about single upstream fetch.

Alex

> 
> Henning
> 
>> diff --git a/meta/classes/build-rep.bbclass
>> b/meta/classes/build-rep.bbclass new file mode 100644
>> index 0000000..ede5a93
>> --- /dev/null
>> +++ b/meta/classes/build-rep.bbclass
>> @@ -0,0 +1,32 @@
>> +# This software is a part of ISAR.
>> +# Copyright (C) 2017 Siemens AG
>> +
>> +python __anonymous() {
>> +    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
>> +    if rep == "0":
>> +        return
>> +
>> +    depsdir = d.getVar('BASE_APT_DIR', True)
>> +    if depsdir is None:
>> +        return
>> +
>> +    depsdir += '/deps/'
>> +
>> +    pn = d.getVar('PN', True)
>> +
>> +    if not os.path.exists(depsdir):
>> +        os.makedirs(depsdir, exist_ok=True)
>> +
>> +    if d.getVar('DEBIAN_DEPENDS', True):
>> +        with open(depsdir + '/' +  pn + '.depends', 'w') as the_file:
>> +            the_file.write(d.getVar('DEBIAN_DEPENDS', True))
>> +            the_file.close
>> +    elif d.getVar('BUILDCHROOT_PREINSTALL', True):
>> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
>> +            the_file.write(d.getVar('BUILDCHROOT_PREINSTALL', True))
>> +            the_file.close
>> +    elif d.getVar('IMAGE_PREINSTALL', True):
>> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as the_file:
>> +            the_file.write(d.getVar('IMAGE_PREINSTALL', True))
>> +            the_file.close
>> +}
>> diff --git a/meta/classes/dpkg-base.bbclass
>> b/meta/classes/dpkg-base.bbclass index 4d220da..bf66e78 100644
>> --- a/meta/classes/dpkg-base.bbclass
>> +++ b/meta/classes/dpkg-base.bbclass
>> @@ -3,6 +3,8 @@
>>   
>>   DEBIAN_DEPENDS ?= ""
>>   
>> +inherit build-rep
>> +
>>   # Add dependency from buildchroot creation
>>   do_build[depends] = "buildchroot:do_build"
>>   
>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>> index e2cb01b..67f5af8 100644
>> --- a/meta/classes/image.bbclass
>> +++ b/meta/classes/image.bbclass
>> @@ -5,6 +5,8 @@ IMAGE_INSTALL ?= ""
>>   IMAGE_TYPE    ?= "ext4-img"
>>   IMAGE_ROOTFS   = "${WORKDIR}/rootfs"
>>   
>> +inherit build-rep
>> +
>>   def get_image_name(d, name_link):
>>       S = d.getVar("IMAGE_ROOTFS", True)
>>       path_link = os.path.join(S, name_link)
>> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb
>> b/meta/recipes-devtools/buildchroot/buildchroot.bb index
>> 51f9d5d..da18231 100644 ---
>> a/meta/recipes-devtools/buildchroot/buildchroot.bb +++
>> b/meta/recipes-devtools/buildchroot/buildchroot.bb @@ -16,6 +16,8 @@
>> SRC_URI = "file://multistrap.conf.in \ file://build.sh"
>>   PV = "1.0"
>>   
>> +inherit build-rep
>> +
>>   BUILDCHROOT_PREINSTALL ?= "gcc \
>>                              make \
>>                              build-essential \
> 

-- 
With best regards,
Alexander Smirnov

ilbers GmbH
Baierbrunner Str. 28c
D-81379 Munich
+49 (89) 122 67 24-0
http://ilbers.de/
Commercial register Munich, HRB 214197
General manager: Baurzhan Ismagulov

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Henning Schild]

Am Fri, 12 Jan 2018 16:29:20 +0300
schrieb Alexander Smirnov <asmirnov@ilbers.de>:

> On 01/12/2018 03:32 PM, Henning Schild wrote:
> > Am Thu, 11 Jan 2018 14:19:38 +0300
> > schrieb Alexander Smirnov <asmirnov@ilbers.de>:
> >   
> >> Add class that helps to implement build reproducibility. It
> >> implements anonymous function that will get all the Debian
> >> dependencies that are needed for current Isar tree.
> >>
> >> Until build reproducibility will be fully implemented, it's
> >> disabled by default. To enable it just set ISAR_BUILD_REP to "1"
> >> in local.conf.
> >>
> >> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> >> ---
> >>   meta-isar/conf/local.conf.sample                 |  6 +++++
> >>   meta-isar/recipes-app/hello/hello.bb             |  2 ++
> >>   meta/classes/build-rep.bbclass                   | 32
> >> ++++++++++++++++++++++++
> >> meta/classes/dpkg-base.bbclass                   |  2 ++
> >> meta/classes/image.bbclass                       |  2 ++
> >> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
> >> changed, 46 insertions(+) create mode 100644
> >> meta/classes/build-rep.bbclass
> >>
> >> diff --git a/meta-isar/conf/local.conf.sample
> >> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
> >> --- a/meta-isar/conf/local.conf.sample
> >> +++ b/meta-isar/conf/local.conf.sample
> >> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
> >>   #
> >>   # Number of attempts to try to get reprepro lock for access to
> >> apt cache REPREPRO_LOCK_ATTEMPTS = "16"
> >> +
> >> +# Isar build reproducibility feature creates local repository
> >> which contains +# copies for all the upstream Debian packages that
> >> could be used to build +# your Isar tree. So fetching them once
> >> will guarantee that all the next Isar +# builds will be
> >> identically. +ISAR_BUILD_REP ?= "0"
> >> diff --git a/meta-isar/recipes-app/hello/hello.bb
> >> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e
> >> 100644 --- a/meta-isar/recipes-app/hello/hello.bb
> >> +++ b/meta-isar/recipes-app/hello/hello.bb
> >> @@ -16,3 +16,5 @@ SRCREV =
> >> "ad7065ecc4840cc436bfcdac427386dbba4ea719" SRC_DIR = "git"
> >>   
> >>   inherit dpkg
> >> +
> >> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"  
> > 
> > If i understand it correctly, this approach is an absolute NoGo.
> > 
> > DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
> > packages. Here you include build-deps of a package that Isar needs
> > to build. And you do so by introducing a copy of a string that is
> > already included in the sources /debian/-folder. In fact you would
> > have to put build and runtime deps into that one variable. And what
> > about packages where the content depends on DISTRO_ARCH and friends?
> > 
> > I think we first need a working solution of two "inherit dpkg"
> > packages actually depending on each-other. After that we will need
> > a real build of the Image to fill the cache. That is the only
> > reliable way to make sure that the repo will contain the runtime
> > and the build deps of all packages. More "sed"-guesswork is not
> > going to do that trick and is only getting us 95% of what we need.
> > And maintaining copies of stuff that is in /debian/ should not be
> > considered.  
> 
> Let's look at this realistically:
> 
> 1. The upstream Debian should be fetched via single operation. If you 
> want to populate the cache during image build - you can't guarantee
> that all the packages were built in the same environment. As
> Christoph wrote, at any time the upstream apt could be updated. So
> the buildchroot and image filesystems should be generated using local
> static apt, otherwise it makes no sense to claim reproducibility.

If the versions change on the way we should end up with two versions of
one package in the cache. When we build from that again we will not get
what we got the first time because image and buildchroot will probably
take the latest version, while in the first run they got different
versions. I think that is fine and we could detect that and declare the
first build as "failed because of upstream change".
You will just have to do that over and over again until you find a
30min window where the mirror did not change, i suspect you will hardly
ever hit the case where the mirror changed between the multistraps.

> 2. Regarding 'debian/control' duplications. According to the idea,
> that 'base-apt' should be generated first of all, in bitbake terms it
> means that:

INHO that idea is wrong, it should be generated as a side-product of an
actual build.
If you want to get this right you will end up reimplementing Debian
internals and keep running after them. That does not sound like a
promising way to go.

>   *.bb:do_build() should depends on base-apt:do_build().
> 
> On the other hand, base-apt:do_build() should already know the full
> list of all deps for *.bb. So for now I see two ways how to implement
> this:
> 
>   - Duplicate 'debian/control' in recipe and share this variable with 
> base-apt (like it's done in this series).
> 
>   - Manually parse 'debian/control'. But in this case base-apt should 
> depend on *.bb:do_unpack. Due to bitbake is 'statically configured' 
> tool, base-apt should know the list of all the recipes, what is also 
> impossible. In previous series I've tried to implement this by
> defining a list of images in some global variable.

"Manuall parse" is a NoGo, just look at the multiple broken versions of
perl code guessing the build-deps ... emulating something that debian
itself just gets right.
 
> 2. Question how to get the list of build and runtime dependencies is 
> still open, I've already asked about this in previous mail. So, the
> goal is to have the full list of packages that should be fetched. 
> 'debian/control' contains very specific format, so I can't feed this 
> line to multistrap/apt-get as it is. I see the following solutions:
>   - Implement custom tool to do this (apt/dpkg uses standard perl
> libs which contains all this stuff).
>   - Manually parse 'debian/control'
>   - Drop the requirement about single upstream fetch.

And again the infamous "manual parse" and even a "custom tool" ... No.

The repo should come out of what the multiple "multistraps" and "apt-get
installs" actually fetch when they are allowed to fetch from the
outside. If you refer to that with "Drop the requirement about ..." i
fully agree.

Henning

> Alex
> 
> > 
> > Henning
> >   
> >> diff --git a/meta/classes/build-rep.bbclass
> >> b/meta/classes/build-rep.bbclass new file mode 100644
> >> index 0000000..ede5a93
> >> --- /dev/null
> >> +++ b/meta/classes/build-rep.bbclass
> >> @@ -0,0 +1,32 @@
> >> +# This software is a part of ISAR.
> >> +# Copyright (C) 2017 Siemens AG
> >> +
> >> +python __anonymous() {
> >> +    rep = d.getVar('ISAR_BUILD_REP', True) or "0"
> >> +    if rep == "0":
> >> +        return
> >> +
> >> +    depsdir = d.getVar('BASE_APT_DIR', True)
> >> +    if depsdir is None:
> >> +        return
> >> +
> >> +    depsdir += '/deps/'
> >> +
> >> +    pn = d.getVar('PN', True)
> >> +
> >> +    if not os.path.exists(depsdir):
> >> +        os.makedirs(depsdir, exist_ok=True)
> >> +
> >> +    if d.getVar('DEBIAN_DEPENDS', True):
> >> +        with open(depsdir + '/' +  pn + '.depends', 'w') as
> >> the_file:
> >> +            the_file.write(d.getVar('DEBIAN_DEPENDS', True))
> >> +            the_file.close
> >> +    elif d.getVar('BUILDCHROOT_PREINSTALL', True):
> >> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as
> >> the_file:
> >> +            the_file.write(d.getVar('BUILDCHROOT_PREINSTALL',
> >> True))
> >> +            the_file.close
> >> +    elif d.getVar('IMAGE_PREINSTALL', True):
> >> +        with open(depsdir + '/' +  pn + '.preinst', 'w') as
> >> the_file:
> >> +            the_file.write(d.getVar('IMAGE_PREINSTALL', True))
> >> +            the_file.close
> >> +}
> >> diff --git a/meta/classes/dpkg-base.bbclass
> >> b/meta/classes/dpkg-base.bbclass index 4d220da..bf66e78 100644
> >> --- a/meta/classes/dpkg-base.bbclass
> >> +++ b/meta/classes/dpkg-base.bbclass
> >> @@ -3,6 +3,8 @@
> >>   
> >>   DEBIAN_DEPENDS ?= ""
> >>   
> >> +inherit build-rep
> >> +
> >>   # Add dependency from buildchroot creation
> >>   do_build[depends] = "buildchroot:do_build"
> >>   
> >> diff --git a/meta/classes/image.bbclass
> >> b/meta/classes/image.bbclass index e2cb01b..67f5af8 100644
> >> --- a/meta/classes/image.bbclass
> >> +++ b/meta/classes/image.bbclass
> >> @@ -5,6 +5,8 @@ IMAGE_INSTALL ?= ""
> >>   IMAGE_TYPE    ?= "ext4-img"
> >>   IMAGE_ROOTFS   = "${WORKDIR}/rootfs"
> >>   
> >> +inherit build-rep
> >> +
> >>   def get_image_name(d, name_link):
> >>       S = d.getVar("IMAGE_ROOTFS", True)
> >>       path_link = os.path.join(S, name_link)
> >> diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb
> >> b/meta/recipes-devtools/buildchroot/buildchroot.bb index
> >> 51f9d5d..da18231 100644 ---
> >> a/meta/recipes-devtools/buildchroot/buildchroot.bb +++
> >> b/meta/recipes-devtools/buildchroot/buildchroot.bb @@ -16,6 +16,8
> >> @@ SRC_URI = "file://multistrap.conf.in \ file://build.sh"
> >>   PV = "1.0"
> >>   
> >> +inherit build-rep
> >> +
> >>   BUILDCHROOT_PREINSTALL ?= "gcc \
> >>                              make \
> >>                              build-essential \  
> >   
> 

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Jan Kiszka]

On 2018-01-12 17:25, [ext] Henning Schild wrote:
> Am Fri, 12 Jan 2018 16:29:20 +0300
> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
> 
>> On 01/12/2018 03:32 PM, Henning Schild wrote:
>>> Am Thu, 11 Jan 2018 14:19:38 +0300
>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>   
>>>> Add class that helps to implement build reproducibility. It
>>>> implements anonymous function that will get all the Debian
>>>> dependencies that are needed for current Isar tree.
>>>>
>>>> Until build reproducibility will be fully implemented, it's
>>>> disabled by default. To enable it just set ISAR_BUILD_REP to "1"
>>>> in local.conf.
>>>>
>>>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>>>> ---
>>>>   meta-isar/conf/local.conf.sample                 |  6 +++++
>>>>   meta-isar/recipes-app/hello/hello.bb             |  2 ++
>>>>   meta/classes/build-rep.bbclass                   | 32
>>>> ++++++++++++++++++++++++
>>>> meta/classes/dpkg-base.bbclass                   |  2 ++
>>>> meta/classes/image.bbclass                       |  2 ++
>>>> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
>>>> changed, 46 insertions(+) create mode 100644
>>>> meta/classes/build-rep.bbclass
>>>>
>>>> diff --git a/meta-isar/conf/local.conf.sample
>>>> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
>>>> --- a/meta-isar/conf/local.conf.sample
>>>> +++ b/meta-isar/conf/local.conf.sample
>>>> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>>>>   #
>>>>   # Number of attempts to try to get reprepro lock for access to
>>>> apt cache REPREPRO_LOCK_ATTEMPTS = "16"
>>>> +
>>>> +# Isar build reproducibility feature creates local repository
>>>> which contains +# copies for all the upstream Debian packages that
>>>> could be used to build +# your Isar tree. So fetching them once
>>>> will guarantee that all the next Isar +# builds will be
>>>> identically. +ISAR_BUILD_REP ?= "0"
>>>> diff --git a/meta-isar/recipes-app/hello/hello.bb
>>>> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e
>>>> 100644 --- a/meta-isar/recipes-app/hello/hello.bb
>>>> +++ b/meta-isar/recipes-app/hello/hello.bb
>>>> @@ -16,3 +16,5 @@ SRCREV =
>>>> "ad7065ecc4840cc436bfcdac427386dbba4ea719" SRC_DIR = "git"
>>>>   
>>>>   inherit dpkg
>>>> +
>>>> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"  
>>>
>>> If i understand it correctly, this approach is an absolute NoGo.
>>>
>>> DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
>>> packages. Here you include build-deps of a package that Isar needs
>>> to build. And you do so by introducing a copy of a string that is
>>> already included in the sources /debian/-folder. In fact you would
>>> have to put build and runtime deps into that one variable. And what
>>> about packages where the content depends on DISTRO_ARCH and friends?
>>>
>>> I think we first need a working solution of two "inherit dpkg"
>>> packages actually depending on each-other. After that we will need
>>> a real build of the Image to fill the cache. That is the only
>>> reliable way to make sure that the repo will contain the runtime
>>> and the build deps of all packages. More "sed"-guesswork is not
>>> going to do that trick and is only getting us 95% of what we need.
>>> And maintaining copies of stuff that is in /debian/ should not be
>>> considered.  
>>
>> Let's look at this realistically:
>>
>> 1. The upstream Debian should be fetched via single operation. If you 
>> want to populate the cache during image build - you can't guarantee
>> that all the packages were built in the same environment. As
>> Christoph wrote, at any time the upstream apt could be updated. So
>> the buildchroot and image filesystems should be generated using local
>> static apt, otherwise it makes no sense to claim reproducibility.
> 
> If the versions change on the way we should end up with two versions of
> one package in the cache. When we build from that again we will not get
> what we got the first time because image and buildchroot will probably
> take the latest version, while in the first run they got different
> versions. I think that is fine and we could detect that and declare the
> first build as "failed because of upstream change".
> You will just have to do that over and over again until you find a
> 30min window where the mirror did not change, i suspect you will hardly
> ever hit the case where the mirror changed between the multistraps.
> 
>> 2. Regarding 'debian/control' duplications. According to the idea,
>> that 'base-apt' should be generated first of all, in bitbake terms it
>> means that:
> 
> INHO that idea is wrong, it should be generated as a side-product of an
> actual build.
> If you want to get this right you will end up reimplementing Debian
> internals and keep running after them. That does not sound like a
> promising way to go.
> 
>>   *.bb:do_build() should depends on base-apt:do_build().
>>
>> On the other hand, base-apt:do_build() should already know the full
>> list of all deps for *.bb. So for now I see two ways how to implement
>> this:
>>
>>   - Duplicate 'debian/control' in recipe and share this variable with 
>> base-apt (like it's done in this series).
>>
>>   - Manually parse 'debian/control'. But in this case base-apt should 
>> depend on *.bb:do_unpack. Due to bitbake is 'statically configured' 
>> tool, base-apt should know the list of all the recipes, what is also 
>> impossible. In previous series I've tried to implement this by
>> defining a list of images in some global variable.
> 
> "Manuall parse" is a NoGo, just look at the multiple broken versions of
> perl code guessing the build-deps ... emulating something that debian
> itself just gets right.
>  
>> 2. Question how to get the list of build and runtime dependencies is 
>> still open, I've already asked about this in previous mail. So, the
>> goal is to have the full list of packages that should be fetched. 
>> 'debian/control' contains very specific format, so I can't feed this 
>> line to multistrap/apt-get as it is. I see the following solutions:
>>   - Implement custom tool to do this (apt/dpkg uses standard perl
>> libs which contains all this stuff).
>>   - Manually parse 'debian/control'
>>   - Drop the requirement about single upstream fetch.
> 
> And again the infamous "manual parse" and even a "custom tool" ... No.
> 
> The repo should come out of what the multiple "multistraps" and "apt-get
> installs" actually fetch when they are allowed to fetch from the
> outside. If you refer to that with "Drop the requirement about ..." i
> fully agree.

I agree with Henning's view. We should exploit Debian for building the
repo and only use bitbake recipes to control and direct Debian tools,
definitely not reimplement them.

Having one build run that both generates outputs and fills that repos
which would then be used on succeeding runs would be ok from user
perspective. We do not necessarily need that strict ordering of
fetch-before-use in the first build.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Benedikt Niedermayr]

Am 14.01.2018 um 17:53 schrieb Jan Kiszka:
> On 2018-01-12 17:25, [ext] Henning Schild wrote:
>> Am Fri, 12 Jan 2018 16:29:20 +0300
>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>
>>> On 01/12/2018 03:32 PM, Henning Schild wrote:
>>>> Am Thu, 11 Jan 2018 14:19:38 +0300
>>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>>    
>>>>> Add class that helps to implement build reproducibility. It
>>>>> implements anonymous function that will get all the Debian
>>>>> dependencies that are needed for current Isar tree.
>>>>>
>>>>> Until build reproducibility will be fully implemented, it's
>>>>> disabled by default. To enable it just set ISAR_BUILD_REP to "1"
>>>>> in local.conf.
>>>>>
>>>>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>>>>> ---
>>>>>    meta-isar/conf/local.conf.sample                 |  6 +++++
>>>>>    meta-isar/recipes-app/hello/hello.bb             |  2 ++
>>>>>    meta/classes/build-rep.bbclass                   | 32
>>>>> ++++++++++++++++++++++++
>>>>> meta/classes/dpkg-base.bbclass                   |  2 ++
>>>>> meta/classes/image.bbclass                       |  2 ++
>>>>> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
>>>>> changed, 46 insertions(+) create mode 100644
>>>>> meta/classes/build-rep.bbclass
>>>>>
>>>>> diff --git a/meta-isar/conf/local.conf.sample
>>>>> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
>>>>> --- a/meta-isar/conf/local.conf.sample
>>>>> +++ b/meta-isar/conf/local.conf.sample
>>>>> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>>>>>    #
>>>>>    # Number of attempts to try to get reprepro lock for access to
>>>>> apt cache REPREPRO_LOCK_ATTEMPTS = "16"
>>>>> +
>>>>> +# Isar build reproducibility feature creates local repository
>>>>> which contains +# copies for all the upstream Debian packages that
>>>>> could be used to build +# your Isar tree. So fetching them once
>>>>> will guarantee that all the next Isar +# builds will be
>>>>> identically. +ISAR_BUILD_REP ?= "0"
>>>>> diff --git a/meta-isar/recipes-app/hello/hello.bb
>>>>> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e
>>>>> 100644 --- a/meta-isar/recipes-app/hello/hello.bb
>>>>> +++ b/meta-isar/recipes-app/hello/hello.bb
>>>>> @@ -16,3 +16,5 @@ SRCREV =
>>>>> "ad7065ecc4840cc436bfcdac427386dbba4ea719" SRC_DIR = "git"
>>>>>    
>>>>>    inherit dpkg
>>>>> +
>>>>> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
>>>> If i understand it correctly, this approach is an absolute NoGo.
>>>>
>>>> DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
>>>> packages. Here you include build-deps of a package that Isar needs
>>>> to build. And you do so by introducing a copy of a string that is
>>>> already included in the sources /debian/-folder. In fact you would
>>>> have to put build and runtime deps into that one variable. And what
>>>> about packages where the content depends on DISTRO_ARCH and friends?
>>>>
>>>> I think we first need a working solution of two "inherit dpkg"
>>>> packages actually depending on each-other. After that we will need
>>>> a real build of the Image to fill the cache. That is the only
>>>> reliable way to make sure that the repo will contain the runtime
>>>> and the build deps of all packages. More "sed"-guesswork is not
>>>> going to do that trick and is only getting us 95% of what we need.
>>>> And maintaining copies of stuff that is in /debian/ should not be
>>>> considered.
>>> Let's look at this realistically:
>>>
>>> 1. The upstream Debian should be fetched via single operation. If you
>>> want to populate the cache during image build - you can't guarantee
>>> that all the packages were built in the same environment. As
>>> Christoph wrote, at any time the upstream apt could be updated. So
>>> the buildchroot and image filesystems should be generated using local
>>> static apt, otherwise it makes no sense to claim reproducibility.
>> If the versions change on the way we should end up with two versions of
>> one package in the cache. When we build from that again we will not get
>> what we got the first time because image and buildchroot will probably
>> take the latest version, while in the first run they got different
>> versions. I think that is fine and we could detect that and declare the
>> first build as "failed because of upstream change".
>> You will just have to do that over and over again until you find a
>> 30min window where the mirror did not change, i suspect you will hardly
>> ever hit the case where the mirror changed between the multistraps.
>>
>>> 2. Regarding 'debian/control' duplications. According to the idea,
>>> that 'base-apt' should be generated first of all, in bitbake terms it
>>> means that:
>> INHO that idea is wrong, it should be generated as a side-product of an
>> actual build.
>> If you want to get this right you will end up reimplementing Debian
>> internals and keep running after them. That does not sound like a
>> promising way to go.
>>
>>>    *.bb:do_build() should depends on base-apt:do_build().
>>>
>>> On the other hand, base-apt:do_build() should already know the full
>>> list of all deps for *.bb. So for now I see two ways how to implement
>>> this:
>>>
>>>    - Duplicate 'debian/control' in recipe and share this variable with
>>> base-apt (like it's done in this series).
>>>
>>>    - Manually parse 'debian/control'. But in this case base-apt should
>>> depend on *.bb:do_unpack. Due to bitbake is 'statically configured'
>>> tool, base-apt should know the list of all the recipes, what is also
>>> impossible. In previous series I've tried to implement this by
>>> defining a list of images in some global variable.
>> "Manuall parse" is a NoGo, just look at the multiple broken versions of
>> perl code guessing the build-deps ... emulating something that debian
>> itself just gets right.
>>   
>>> 2. Question how to get the list of build and runtime dependencies is
>>> still open, I've already asked about this in previous mail. So, the
>>> goal is to have the full list of packages that should be fetched.
>>> 'debian/control' contains very specific format, so I can't feed this
>>> line to multistrap/apt-get as it is. I see the following solutions:
>>>    - Implement custom tool to do this (apt/dpkg uses standard perl
>>> libs which contains all this stuff).
>>>    - Manually parse 'debian/control'
>>>    - Drop the requirement about single upstream fetch.
>> And again the infamous "manual parse" and even a "custom tool" ... No.
>>
>> The repo should come out of what the multiple "multistraps" and "apt-get
>> installs" actually fetch when they are allowed to fetch from the
>> outside. If you refer to that with "Drop the requirement about ..." i
>> fully agree.
> I agree with Henning's view. We should exploit Debian for building the
> repo and only use bitbake recipes to control and direct Debian tools,
> definitely not reimplement them.
>
> Having one build run that both generates outputs and fills that repos
> which would then be used on succeeding runs would be ok from user
> perspective. We do not necessarily need that strict ordering of
> fetch-before-use in the first build.
>
> Jan
>
I do also agree with Henning's view.


It would be better not struggling with those complexity within Isar.
Maybe I'm a bit late. But is reprepro really the best solution for 
implementing the cache?
Let's assume we implement it like Henning mentioned already. We let Isar 
generate its image and add all required packages to the local repository.


Then I see a big problem when adding new recipes to Isar, which will extend
the local apt repository with maybe new versions of a dependent package. 
You never know that before.
In this case the new dependencies can trigger a bunch replacement of the 
old packages, and the old stable state of local repo is gone.
The solution may be to create snapshots of the current local repository, 
but reprepro is IMHO therefore not the best solution:
https://serverfault.com/questions/489504/using-snapshots-with-reprepro-to-enable-rollbacks 

I think "aptly" would be much better here, since aptly gives you the 
possibility to create partial snapshots of new package versions.
It also comes along with snapshot merging and lot of things Isar 
currently doesn't need, but maybe will sometime in the future.


I imagine the apt-cache as a real cache. Means this cache should be able 
to be the interface between Isar and downloading the packages.
So the cache should download the package (and dependencies) into the 
cache and Isar installs those packages from the cache instead from the 
remote repository.
I have not tested that with aptly, but the link looks like it can 
achieve that:
https://www.aptly.info/tutorial/pull/
When aptly is capable of downloading single packages (and dependencies) 
from a remote repo, then i strongly recommend using this tool instead
of reprepro.


Regards

Benni

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Baurzhan Ismagulov]

On Sun, Jan 14, 2018 at 05:53:03PM +0100, Jan Kiszka wrote:
> I agree with Henning's view. We should exploit Debian for building the
> repo and only use bitbake recipes to control and direct Debian tools,
> definitely not reimplement them.
> 
> Having one build run that both generates outputs and fills that repos
> which would then be used on succeeding runs would be ok from user
> perspective. We do not necessarily need that strict ordering of
> fetch-before-use in the first build.

It's a trade-off between perfectly solving the by-product way vs. solving the
up-front way with TODOs for the next steps.


My original vision has been to have a separate partial mirroring tool that
understands Isar metadata (recipes and / or Debian packages).

1. Conceptually, I dislike the by-product solution because in the projects,
   repo maintenance is often done by configuration managers. They don't build
   the code, they just make the mirror available (with Isar's predecessor,
   those were complete mirrors made with debmirror). Project developers switch
   from one snapshot to another and adjust the code if necessary. Requiring CMs
   to build the code is not optimal and, in my opinion, should not be done. We
   should provide tools and people should decide whether to combine them or
   not.

   Providing separate mirroring is also the right step towards offline work and
   SDK as supported by Yocto.

2. Mirrors are maintained per image, because two products in the same repo may
   have different release timelines. That is why we need to have an explicit
   list of images to produce the mirror for. This could be addressed with both
   bitbake-based approaches, by-product and up-front.

3. Projects update from time to time, not on every build. This could be done
   with both approaches through turning off mirroring in local.conf. With a
   separate tool, it would be more convenient to use.

4. Finally, there is a question of partial updates, e.g., update only glib and,
   optionally, its dependencies, dependents, and conflicting packages. This is
   not addressed by either of bitbake-based solutions.


To create such a tool, we would have to:

1. Understand Isar metadata.

2. Implement mirroring.

To avoid messing with (1) at this time, we used bitbake for now. The goal is to
implement mirroring quickly and get experience with it, because important
details may require adjusting the vision.


There are also technical issues with the by-product solution:

1. Complicated and error-prone workflow
2. Unnecessary build repeat burden for the project developer
3. Intertwined, inseparable mirror and build
4. Unpredictable builds, esp. if sid is used (usual for early development)


The following technical issues have been raised regarding the two up-front
implementations:

1. DEBIAN_DEPENDS mixes Build-Depends and Depends.

   I agree that this is not optimal. This could be solved by separating
   DEBIAN_DEPENDS and DEBIAN_BUILD_DEPENDS. This itself would be a temporary
   solution till we implement debian/control backend for bitbake (as an
   alternative to .bb backend).

2. We first need a working solution of two "inherit dpkg" packages actually
   depending on each-other.

   An intermediate solution is available in Isar's predecessor. The approach is
   to provide DEPENDS in recipes to ensure the right do_build order, and to
   install the development packages into buildchroot. Of course, in the future
   that should be extended with adding the built packages to apt and moving to
   debian/control backend.

   That said, it's a matter of priority -- reproducibility and wic were at the
   top of Jan's wishlist. FWIW, Alex already has an implementation for own
   package dependencies and will post it.

3. Reimplementing Debian tools and constant catching up with upstream.

   Some conceptual points first. As already discussed in the context of wic
   last year, we strive for solving project problems today and merging with
   upstream after we get experience with the feature, as opposed to developing
   huge series solving all problems of the world and pushing them upstream
   first. This delayed feedback loop works well e.g. for C and C++
   standardization. You see, now you promise a better implementation with less
   or no changes to wic; it may be well worth waiting with upstreaming. This is
   not to say we want to leave the problems in the code; we just want to choose
   what problem we tackle at which point of time.

   Another point is, we don't actually reimplement Debian tools. If a tool that
   we need existed, we would have used it, no? But they don't. So, we can
   extend an upstream one, or we can implement our own. For a PoC, we choose
   whatever is faster. Merging with upstream is a TODO that we deliberately add
   to our list. I agree that in this way, we've accumulated quite a number of
   hacks; we are going to resolve them.

   That said, we shouldn't shy away from creating and maintaining new tools if
   it's a better way of solving the problem. Debian describes the format of the
   files, and many tools provide similar functionality. Many Linux features
   have been maintained as patches for years before being accepted to the
   mainline.

   Returning to the issue at hand: I agree that parsing debian/control manually
   is not optimal. Isar already does that in other places; upstreaming it to
   some tool is a TODO that should not block moving forward on reproducibility.
   We expect changes in this area, so even the choice of the tool to merge
   depends on the workflow we'd end up with.


Up-front mirroring was Christian's requirement that noone objected to at the
time [1]. Considering all of this:

1. I suggest to clean up the first series with the list of images due to the
   use case #2, and merge it.

2. I'd rather not go for the second series, to avoid opening a new can of worms
   w.r.t. DEBIAN_DEPENDS, unless there are serious technical reasons in its
   favor.

3. We have always been aware of the issues and are working on them.


With kind regards,
Baurzhan.


References:

1. https://groups.google.com/d/msg/isar-users/4PVondzZryk/qFcYq1qqAQAJ

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Jan Kiszka]

On 2018-01-23 12:50, Baurzhan Ismagulov wrote:
> On Sun, Jan 14, 2018 at 05:53:03PM +0100, Jan Kiszka wrote:
>> I agree with Henning's view. We should exploit Debian for building the
>> repo and only use bitbake recipes to control and direct Debian tools,
>> definitely not reimplement them.
>>
>> Having one build run that both generates outputs and fills that repos
>> which would then be used on succeeding runs would be ok from user
>> perspective. We do not necessarily need that strict ordering of
>> fetch-before-use in the first build.
> 
> It's a trade-off between perfectly solving the by-product way vs. solving the
> up-front way with TODOs for the next steps.
> 
> 
> My original vision has been to have a separate partial mirroring tool that
> understands Isar metadata (recipes and / or Debian packages).
> 
> 1. Conceptually, I dislike the by-product solution because in the projects,
>    repo maintenance is often done by configuration managers. They don't build
>    the code, they just make the mirror available (with Isar's predecessor,
>    those were complete mirrors made with debmirror). Project developers switch
>    from one snapshot to another and adjust the code if necessary. Requiring CMs
>    to build the code is not optimal and, in my opinion, should not be done. We
>    should provide tools and people should decide whether to combine them or
>    not.

Configuation managers are not our primary concern. Addressing all their
"needs" (or lack of technical experience) will restrict us too much.
Let's do something for today's devops first and address that special
case on top - locally.

> 
>    Providing separate mirroring is also the right step towards offline work and
>    SDK as supported by Yocto.

I don't see how having online connection for the first run and then
being able to rebuild offline afterwards is a problem, or something that
restricts SDK builds.

> 
> 2. Mirrors are maintained per image, because two products in the same repo may
>    have different release timelines. That is why we need to have an explicit
>    list of images to produce the mirror for. This could be addressed with both
>    bitbake-based approaches, by-product and up-front.

Again: user experience. The common case is that you do not mix
reproducible and non-reproducible images in a single build run. So you
also want to control that with a single, general knob, and not by
duplicating the list that you already pass to bitbake.

> 
> 3. Projects update from time to time, not on every build. This could be done
>    with both approaches through turning off mirroring in local.conf. With a
>    separate tool, it would be more convenient to use.

Only if that tool is as robust as the build itself. I and also Henning
have major concerns that this will not be the case, thus our requirement
to avoid that logic duplication and fragility.

> 
> 4. Finally, there is a question of partial updates, e.g., update only glib and,
>    optionally, its dependencies, dependents, and conflicting packages. This is
>    not addressed by either of bitbake-based solutions.

That could probably be solved with an update "mask", a list of packages
passed to a build that should be pulled from extern vs. from the
internal mirror.

> 
> 
> To create such a tool, we would have to:
> 
> 1. Understand Isar metadata.
> 
> 2. Implement mirroring.
> 
> To avoid messing with (1) at this time, we used bitbake for now. The goal is to
> implement mirroring quickly and get experience with it, because important
> details may require adjusting the vision.
> 
> 
> There are also technical issues with the by-product solution:
> 
> 1. Complicated and error-prone workflow
> 2. Unnecessary build repeat burden for the project developer

There is not need to repeat the build to get a mirror.

> 3. Intertwined, inseparable mirror and build
> 4. Unpredictable builds, esp. if sid is used (usual for early development)
> 
> 
> The following technical issues have been raised regarding the two up-front
> implementations:
> 
> 1. DEBIAN_DEPENDS mixes Build-Depends and Depends.
> 
>    I agree that this is not optimal. This could be solved by separating
>    DEBIAN_DEPENDS and DEBIAN_BUILD_DEPENDS. This itself would be a temporary
>    solution till we implement debian/control backend for bitbake (as an
>    alternative to .bb backend).
> 
> 2. We first need a working solution of two "inherit dpkg" packages actually
>    depending on each-other.
> 
>    An intermediate solution is available in Isar's predecessor. The approach is
>    to provide DEPENDS in recipes to ensure the right do_build order, and to
>    install the development packages into buildchroot. Of course, in the future
>    that should be extended with adding the built packages to apt and moving to
>    debian/control backend.
> 
>    That said, it's a matter of priority -- reproducibility and wic were at the
>    top of Jan's wishlist. FWIW, Alex already has an implementation for own
>    package dependencies and will post it.

wic was addressed by Henning, you can focus on reviews.

> 
> 3. Reimplementing Debian tools and constant catching up with upstream.
> 
>    Some conceptual points first. As already discussed in the context of wic
>    last year, we strive for solving project problems today and merging with
>    upstream after we get experience with the feature, as opposed to developing
>    huge series solving all problems of the world and pushing them upstream
>    first. This delayed feedback loop works well e.g. for C and C++
>    standardization. You see, now you promise a better implementation with less
>    or no changes to wic; it may be well worth waiting with upstreaming. This is
>    not to say we want to leave the problems in the code; we just want to choose
>    what problem we tackle at which point of time.
> 
>    Another point is, we don't actually reimplement Debian tools. If a tool that
>    we need existed, we would have used it, no? But they don't. So, we can
>    extend an upstream one, or we can implement our own. For a PoC, we choose
>    whatever is faster. Merging with upstream is a TODO that we deliberately add
>    to our list. I agree that in this way, we've accumulated quite a number of
>    hacks; we are going to resolve them.
> 
>    That said, we shouldn't shy away from creating and maintaining new tools if
>    it's a better way of solving the problem. Debian describes the format of the
>    files, and many tools provide similar functionality. Many Linux features
>    have been maintained as patches for years before being accepted to the
>    mainline.
> 
>    Returning to the issue at hand: I agree that parsing debian/control manually
>    is not optimal. Isar already does that in other places; upstreaming it to
>    some tool is a TODO that should not block moving forward on reproducibility.
>    We expect changes in this area, so even the choice of the tool to merge
>    depends on the workflow we'd end up with.
> 
> 
> Up-front mirroring was Christian's requirement that noone objected to at the
> time [1]. Considering all of this:
> 
> 1. I suggest to clean up the first series with the list of images due to the
>    use case #2, and merge it.
> 
> 2. I'd rather not go for the second series, to avoid opening a new can of worms
>    w.r.t. DEBIAN_DEPENDS, unless there are serious technical reasons in its
>    favor.
> 
> 3. We have always been aware of the issues and are working on them.
> 

This is a strategical project decision because it affect the user
experience and the way recipes are written. Despite the project needs,
we must not hurry quick intermediate solutions (unless the are truly
orthogonal to the recipes).

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Christian Storm]

Hi all,

> [...]
> My original vision has been to have a separate partial mirroring tool that
> understands Isar metadata (recipes and / or Debian packages).
> 
> 1. Conceptually, I dislike the by-product solution because in the projects,
>    repo maintenance is often done by configuration managers. They don't build
>    the code, they just make the mirror available (with Isar's predecessor,
>    those were complete mirrors made with debmirror). Project developers switch
>    from one snapshot to another and adjust the code if necessary. Requiring CMs
>    to build the code is not optimal and, in my opinion, should not be done. We
>    should provide tools and people should decide whether to combine them or
>    not.
> 
>    Providing separate mirroring is also the right step towards offline work and
>    SDK as supported by Yocto.
> 
> 2. Mirrors are maintained per image, because two products in the same repo may
>    have different release timelines. That is why we need to have an explicit
>    list of images to produce the mirror for. This could be addressed with both
>    bitbake-based approaches, by-product and up-front.
> 
> 3. Projects update from time to time, not on every build. This could be done
>    with both approaches through turning off mirroring in local.conf. With a
>    separate tool, it would be more convenient to use.
> 
> 4. Finally, there is a question of partial updates, e.g., update only glib and,
>    optionally, its dependencies, dependents, and conflicting packages. This is
>    not addressed by either of bitbake-based solutions.
> 
> 
> To create such a tool, we would have to:
> 
> 1. Understand Isar metadata.
> 
> 2. Implement mirroring.
> 
> To avoid messing with (1) at this time, we used bitbake for now. The goal is to
> implement mirroring quickly and get experience with it, because important
> details may require adjusting the vision.

Then this qualifies for a proof-of-concept on which we may discuss,
technically, I would say. Whether this "mirroring tool" belongs into
ISAR's core or should be an external tool that's suited towards ISAR is
a strategic architectural decision. I'm in favor of the former.


> There are also technical issues with the by-product solution:
> 
> 1. Complicated and error-prone workflow
> 2. Unnecessary build repeat burden for the project developer
> 3. Intertwined, inseparable mirror and build
> 4. Unpredictable builds, esp. if sid is used (usual for early development)
> 
> 
> The following technical issues have been raised regarding the two up-front
> implementations:
> 
> 1. DEBIAN_DEPENDS mixes Build-Depends and Depends.
> 
>    I agree that this is not optimal. This could be solved by separating
>    DEBIAN_DEPENDS and DEBIAN_BUILD_DEPENDS. This itself would be a temporary
>    solution till we implement debian/control backend for bitbake (as an
>    alternative to .bb backend).

Well, that would deviate from bitbake syntax and "look-and-feel" more
than it is already...


> 2. We first need a working solution of two "inherit dpkg" packages actually
>    depending on each-other.
> 
>    An intermediate solution is available in Isar's predecessor. The approach is
>    to provide DEPENDS in recipes to ensure the right do_build order, and to
>    install the development packages into buildchroot. Of course, in the future
>    that should be extended with adding the built packages to apt and moving to
>    debian/control backend.
> 
>    That said, it's a matter of priority -- reproducibility and wic were at the
>    top of Jan's wishlist. FWIW, Alex already has an implementation for own
>    package dependencies and will post it.
> 
> 3. Reimplementing Debian tools and constant catching up with upstream.
> 
>    Some conceptual points first. As already discussed in the context of wic
>    last year, we strive for solving project problems today and merging with
>    upstream after we get experience with the feature, as opposed to developing
>    huge series solving all problems of the world and pushing them upstream
>    first. This delayed feedback loop works well e.g. for C and C++
>    standardization. You see, now you promise a better implementation with less
>    or no changes to wic; it may be well worth waiting with upstreaming. This is
>    not to say we want to leave the problems in the code; we just want to choose
>    what problem we tackle at which point of time.
> 
>    Another point is, we don't actually reimplement Debian tools. If a tool that
>    we need existed, we would have used it, no? But they don't. So, we can
>    extend an upstream one, or we can implement our own. For a PoC, we choose
>    whatever is faster. Merging with upstream is a TODO that we deliberately add
>    to our list. I agree that in this way, we've accumulated quite a number of
>    hacks; we are going to resolve them.
> 
>    That said, we shouldn't shy away from creating and maintaining new tools if
>    it's a better way of solving the problem. Debian describes the format of the
>    files, and many tools provide similar functionality. Many Linux features
>    have been maintained as patches for years before being accepted to the
>    mainline.
> 
>    Returning to the issue at hand: I agree that parsing debian/control manually
>    is not optimal. Isar already does that in other places; upstreaming it to
>    some tool is a TODO that should not block moving forward on reproducibility.
>    We expect changes in this area, so even the choice of the tool to merge
>    depends on the workflow we'd end up with.
> 
> 
> Up-front mirroring was Christian's requirement that noone objected to at the
> time [1]. Considering all of this:
> [...]

Well, recalling what I wrote back in November, I don't read a
requirement, literally, on upfront mirroring. I explicitly mentioned
this to be my brain dump of ideas and wrap up of the discussion so far
in order to revive the discussion on the topic and to foster the
implementation of this feature which I still very much like to see
become upstream -master reality. 
I did explicitly *not* request a particular implementation -- as I tried
to make this clear by using "Regardless/irrespective of the technical
implementation" multiple times -- in order to find the best solution,
which is contradictory to requesting some concrete implementation of
the feature. Maybe this was not explicit enough though...
That said, upfront mirroring may be *one* solution -- and maybe the most
straightforward one -- to the Debian-inherent window of vulnerability
problem I described. There are certainly others... Finding and
elaborating on those was my intention, not dictating a solution.





Kind regards,
   Christian

-- 
Dr. Christian Storm
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Otto-Hahn-Ring 6, 81739 M�nchen, Germany

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Baurzhan Ismagulov]

On Tue, Jan 23, 2018 at 02:02:24PM +0100, Jan Kiszka wrote:
> >    That said, it's a matter of priority -- reproducibility and wic were at the
> >    top of Jan's wishlist. FWIW, Alex already has an implementation for own
> >    package dependencies and will post it.
> 
> wic was addressed by Henning, you can focus on reviews.

I had understood this, thus "were" [when we started working on that].


> This is a strategical project decision because it affect the user
> experience and the way recipes are written.

Then, I'd suggest to continue the discussion when Henning is back.


> Despite the project needs,
> we must not hurry quick intermediate solutions (unless the are truly
> orthogonal to the recipes).

Intertwined mirror and build is an inferior approach. So, my point is not to
invest in it, because we have a working PoC of a better solution. We can
address outstanding technical issues before or after merging -- the question
being one line in debian/control.

To make it clear, "quick" isn't about "hurry", it's about work breakdown and
moving in small steps. Having a PoC and hands-on experience with it is crucial
for coming up with the right concept at the end.


With kind regards,
Baurzhan.

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Jan Kiszka]

On 2018-01-19 22:23, 'Benedikt Niedermayr' via isar-users wrote:
> Am 14.01.2018 um 17:53 schrieb Jan Kiszka:
>> On 2018-01-12 17:25, [ext] Henning Schild wrote:
>>> Am Fri, 12 Jan 2018 16:29:20 +0300
>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>
>>>> On 01/12/2018 03:32 PM, Henning Schild wrote:
>>>>> Am Thu, 11 Jan 2018 14:19:38 +0300
>>>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>>>   
>>>>>> Add class that helps to implement build reproducibility. It
>>>>>> implements anonymous function that will get all the Debian
>>>>>> dependencies that are needed for current Isar tree.
>>>>>>
>>>>>> Until build reproducibility will be fully implemented, it's
>>>>>> disabled by default. To enable it just set ISAR_BUILD_REP to "1"
>>>>>> in local.conf.
>>>>>>
>>>>>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>>>>>> ---
>>>>>>    meta-isar/conf/local.conf.sample                 |  6 +++++
>>>>>>    meta-isar/recipes-app/hello/hello.bb             |  2 ++
>>>>>>    meta/classes/build-rep.bbclass                   | 32
>>>>>> ++++++++++++++++++++++++
>>>>>> meta/classes/dpkg-base.bbclass                   |  2 ++
>>>>>> meta/classes/image.bbclass                       |  2 ++
>>>>>> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
>>>>>> changed, 46 insertions(+) create mode 100644
>>>>>> meta/classes/build-rep.bbclass
>>>>>>
>>>>>> diff --git a/meta-isar/conf/local.conf.sample
>>>>>> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
>>>>>> --- a/meta-isar/conf/local.conf.sample
>>>>>> +++ b/meta-isar/conf/local.conf.sample
>>>>>> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>>>>>>    #
>>>>>>    # Number of attempts to try to get reprepro lock for access to
>>>>>> apt cache REPREPRO_LOCK_ATTEMPTS = "16"
>>>>>> +
>>>>>> +# Isar build reproducibility feature creates local repository
>>>>>> which contains +# copies for all the upstream Debian packages that
>>>>>> could be used to build +# your Isar tree. So fetching them once
>>>>>> will guarantee that all the next Isar +# builds will be
>>>>>> identically. +ISAR_BUILD_REP ?= "0"
>>>>>> diff --git a/meta-isar/recipes-app/hello/hello.bb
>>>>>> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e
>>>>>> 100644 --- a/meta-isar/recipes-app/hello/hello.bb
>>>>>> +++ b/meta-isar/recipes-app/hello/hello.bb
>>>>>> @@ -16,3 +16,5 @@ SRCREV =
>>>>>> "ad7065ecc4840cc436bfcdac427386dbba4ea719" SRC_DIR = "git"
>>>>>>       inherit dpkg
>>>>>> +
>>>>>> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
>>>>> If i understand it correctly, this approach is an absolute NoGo.
>>>>>
>>>>> DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
>>>>> packages. Here you include build-deps of a package that Isar needs
>>>>> to build. And you do so by introducing a copy of a string that is
>>>>> already included in the sources /debian/-folder. In fact you would
>>>>> have to put build and runtime deps into that one variable. And what
>>>>> about packages where the content depends on DISTRO_ARCH and friends?
>>>>>
>>>>> I think we first need a working solution of two "inherit dpkg"
>>>>> packages actually depending on each-other. After that we will need
>>>>> a real build of the Image to fill the cache. That is the only
>>>>> reliable way to make sure that the repo will contain the runtime
>>>>> and the build deps of all packages. More "sed"-guesswork is not
>>>>> going to do that trick and is only getting us 95% of what we need.
>>>>> And maintaining copies of stuff that is in /debian/ should not be
>>>>> considered.
>>>> Let's look at this realistically:
>>>>
>>>> 1. The upstream Debian should be fetched via single operation. If you
>>>> want to populate the cache during image build - you can't guarantee
>>>> that all the packages were built in the same environment. As
>>>> Christoph wrote, at any time the upstream apt could be updated. So
>>>> the buildchroot and image filesystems should be generated using local
>>>> static apt, otherwise it makes no sense to claim reproducibility.
>>> If the versions change on the way we should end up with two versions of
>>> one package in the cache. When we build from that again we will not get
>>> what we got the first time because image and buildchroot will probably
>>> take the latest version, while in the first run they got different
>>> versions. I think that is fine and we could detect that and declare the
>>> first build as "failed because of upstream change".
>>> You will just have to do that over and over again until you find a
>>> 30min window where the mirror did not change, i suspect you will hardly
>>> ever hit the case where the mirror changed between the multistraps.
>>>
>>>> 2. Regarding 'debian/control' duplications. According to the idea,
>>>> that 'base-apt' should be generated first of all, in bitbake terms it
>>>> means that:
>>> INHO that idea is wrong, it should be generated as a side-product of an
>>> actual build.
>>> If you want to get this right you will end up reimplementing Debian
>>> internals and keep running after them. That does not sound like a
>>> promising way to go.
>>>
>>>>    *.bb:do_build() should depends on base-apt:do_build().
>>>>
>>>> On the other hand, base-apt:do_build() should already know the full
>>>> list of all deps for *.bb. So for now I see two ways how to implement
>>>> this:
>>>>
>>>>    - Duplicate 'debian/control' in recipe and share this variable with
>>>> base-apt (like it's done in this series).
>>>>
>>>>    - Manually parse 'debian/control'. But in this case base-apt should
>>>> depend on *.bb:do_unpack. Due to bitbake is 'statically configured'
>>>> tool, base-apt should know the list of all the recipes, what is also
>>>> impossible. In previous series I've tried to implement this by
>>>> defining a list of images in some global variable.
>>> "Manuall parse" is a NoGo, just look at the multiple broken versions of
>>> perl code guessing the build-deps ... emulating something that debian
>>> itself just gets right.
>>>  
>>>> 2. Question how to get the list of build and runtime dependencies is
>>>> still open, I've already asked about this in previous mail. So, the
>>>> goal is to have the full list of packages that should be fetched.
>>>> 'debian/control' contains very specific format, so I can't feed this
>>>> line to multistrap/apt-get as it is. I see the following solutions:
>>>>    - Implement custom tool to do this (apt/dpkg uses standard perl
>>>> libs which contains all this stuff).
>>>>    - Manually parse 'debian/control'
>>>>    - Drop the requirement about single upstream fetch.
>>> And again the infamous "manual parse" and even a "custom tool" ... No.
>>>
>>> The repo should come out of what the multiple "multistraps" and "apt-get
>>> installs" actually fetch when they are allowed to fetch from the
>>> outside. If you refer to that with "Drop the requirement about ..." i
>>> fully agree.
>> I agree with Henning's view. We should exploit Debian for building the
>> repo and only use bitbake recipes to control and direct Debian tools,
>> definitely not reimplement them.
>>
>> Having one build run that both generates outputs and fills that repos
>> which would then be used on succeeding runs would be ok from user
>> perspective. We do not necessarily need that strict ordering of
>> fetch-before-use in the first build.
>>
>> Jan
>>
> I do also agree with Henning's view.
> 
> 
> It would be better not struggling with those complexity within Isar.
> Maybe I'm a bit late. But is reprepro really the best solution for
> implementing the cache?
> Let's assume we implement it like Henning mentioned already. We let Isar
> generate its image and add all required packages to the local repository.
> 
> 
> Then I see a big problem when adding new recipes to Isar, which will extend
> the local apt repository with maybe new versions of a dependent package.
> You never know that before.
> In this case the new dependencies can trigger a bunch replacement of the
> old packages, and the old stable state of local repo is gone.
> The solution may be to create snapshots of the current local repository,
> but reprepro is IMHO therefore not the best solution:
> https://serverfault.com/questions/489504/using-snapshots-with-reprepro-to-enable-rollbacks
> 
> I think "aptly" would be much better here, since aptly gives you the
> possibility to create partial snapshots of new package versions.
> It also comes along with snapshot merging and lot of things Isar
> currently doesn't need, but maybe will sometime in the future.
> 
> 
> I imagine the apt-cache as a real cache. Means this cache should be able
> to be the interface between Isar and downloading the packages.
> So the cache should download the package (and dependencies) into the
> cache and Isar installs those packages from the cache instead from the
> remote repository.
> I have not tested that with aptly, but the link looks like it can
> achieve that:
> https://www.aptly.info/tutorial/pull/
> When aptly is capable of downloading single packages (and dependencies)
> from a remote repo, then i strongly recommend using this tool instead
> of reprepro.
> 

I also started to look into aptly, which I had on my notes for a while.
One reason is that it could act as a manual workaround for the missing
version-freezing support in Isar (pull into aptly repo, take a snapshot,
and then build from it with Isar).

Another is that we also need some tooling to manage mirrors and
snapshots on the long run, and reprepro, TBHO, smells strange. On first
glance, it looks to me like the multistrap of the repo management tools:
once a cool thing, but now unmaintained.

Did you dig further into aptly? I'm currently wondering how it
internally manages snapshots, if it does proper deduplication and how
much of that can be preserved when publishing for consumption by, say,
Isar...

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Benedikt Niedermayr]

Am 24.01.2018 um 19:48 schrieb Jan Kiszka:
> On 2018-01-19 22:23, 'Benedikt Niedermayr' via isar-users wrote:
>> Am 14.01.2018 um 17:53 schrieb Jan Kiszka:
>>> On 2018-01-12 17:25, [ext] Henning Schild wrote:
>>>> Am Fri, 12 Jan 2018 16:29:20 +0300
>>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>>
>>>>> On 01/12/2018 03:32 PM, Henning Schild wrote:
>>>>>> Am Thu, 11 Jan 2018 14:19:38 +0300
>>>>>> schrieb Alexander Smirnov <asmirnov@ilbers.de>:
>>>>>>    
>>>>>>> Add class that helps to implement build reproducibility. It
>>>>>>> implements anonymous function that will get all the Debian
>>>>>>> dependencies that are needed for current Isar tree.
>>>>>>>
>>>>>>> Until build reproducibility will be fully implemented, it's
>>>>>>> disabled by default. To enable it just set ISAR_BUILD_REP to "1"
>>>>>>> in local.conf.
>>>>>>>
>>>>>>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>>>>>>> ---
>>>>>>>     meta-isar/conf/local.conf.sample                 |  6 +++++
>>>>>>>     meta-isar/recipes-app/hello/hello.bb             |  2 ++
>>>>>>>     meta/classes/build-rep.bbclass                   | 32
>>>>>>> ++++++++++++++++++++++++
>>>>>>> meta/classes/dpkg-base.bbclass                   |  2 ++
>>>>>>> meta/classes/image.bbclass                       |  2 ++
>>>>>>> meta/recipes-devtools/buildchroot/buildchroot.bb |  2 ++ 6 files
>>>>>>> changed, 46 insertions(+) create mode 100644
>>>>>>> meta/classes/build-rep.bbclass
>>>>>>>
>>>>>>> diff --git a/meta-isar/conf/local.conf.sample
>>>>>>> b/meta-isar/conf/local.conf.sample index 660958f..45b8995 100644
>>>>>>> --- a/meta-isar/conf/local.conf.sample
>>>>>>> +++ b/meta-isar/conf/local.conf.sample
>>>>>>> @@ -162,3 +162,9 @@ BB_NUMBER_THREADS = "4"
>>>>>>>     #
>>>>>>>     # Number of attempts to try to get reprepro lock for access to
>>>>>>> apt cache REPREPRO_LOCK_ATTEMPTS = "16"
>>>>>>> +
>>>>>>> +# Isar build reproducibility feature creates local repository
>>>>>>> which contains +# copies for all the upstream Debian packages that
>>>>>>> could be used to build +# your Isar tree. So fetching them once
>>>>>>> will guarantee that all the next Isar +# builds will be
>>>>>>> identically. +ISAR_BUILD_REP ?= "0"
>>>>>>> diff --git a/meta-isar/recipes-app/hello/hello.bb
>>>>>>> b/meta-isar/recipes-app/hello/hello.bb index 44b8bc3..fafda2e
>>>>>>> 100644 --- a/meta-isar/recipes-app/hello/hello.bb
>>>>>>> +++ b/meta-isar/recipes-app/hello/hello.bb
>>>>>>> @@ -16,3 +16,5 @@ SRCREV =
>>>>>>> "ad7065ecc4840cc436bfcdac427386dbba4ea719" SRC_DIR = "git"
>>>>>>>        inherit dpkg
>>>>>>> +
>>>>>>> +DEBIAN_DEPENDS = "debhelper (>= 9), autotools-dev"
>>>>>> If i understand it correctly, this approach is an absolute NoGo.
>>>>>>
>>>>>> DEBIAN_DEPENDS as it is used today are runtime deps of pre-built
>>>>>> packages. Here you include build-deps of a package that Isar needs
>>>>>> to build. And you do so by introducing a copy of a string that is
>>>>>> already included in the sources /debian/-folder. In fact you would
>>>>>> have to put build and runtime deps into that one variable. And what
>>>>>> about packages where the content depends on DISTRO_ARCH and friends?
>>>>>>
>>>>>> I think we first need a working solution of two "inherit dpkg"
>>>>>> packages actually depending on each-other. After that we will need
>>>>>> a real build of the Image to fill the cache. That is the only
>>>>>> reliable way to make sure that the repo will contain the runtime
>>>>>> and the build deps of all packages. More "sed"-guesswork is not
>>>>>> going to do that trick and is only getting us 95% of what we need.
>>>>>> And maintaining copies of stuff that is in /debian/ should not be
>>>>>> considered.
>>>>> Let's look at this realistically:
>>>>>
>>>>> 1. The upstream Debian should be fetched via single operation. If you
>>>>> want to populate the cache during image build - you can't guarantee
>>>>> that all the packages were built in the same environment. As
>>>>> Christoph wrote, at any time the upstream apt could be updated. So
>>>>> the buildchroot and image filesystems should be generated using local
>>>>> static apt, otherwise it makes no sense to claim reproducibility.
>>>> If the versions change on the way we should end up with two versions of
>>>> one package in the cache. When we build from that again we will not get
>>>> what we got the first time because image and buildchroot will probably
>>>> take the latest version, while in the first run they got different
>>>> versions. I think that is fine and we could detect that and declare the
>>>> first build as "failed because of upstream change".
>>>> You will just have to do that over and over again until you find a
>>>> 30min window where the mirror did not change, i suspect you will hardly
>>>> ever hit the case where the mirror changed between the multistraps.
>>>>
>>>>> 2. Regarding 'debian/control' duplications. According to the idea,
>>>>> that 'base-apt' should be generated first of all, in bitbake terms it
>>>>> means that:
>>>> INHO that idea is wrong, it should be generated as a side-product of an
>>>> actual build.
>>>> If you want to get this right you will end up reimplementing Debian
>>>> internals and keep running after them. That does not sound like a
>>>> promising way to go.
>>>>
>>>>>     *.bb:do_build() should depends on base-apt:do_build().
>>>>>
>>>>> On the other hand, base-apt:do_build() should already know the full
>>>>> list of all deps for *.bb. So for now I see two ways how to implement
>>>>> this:
>>>>>
>>>>>     - Duplicate 'debian/control' in recipe and share this variable with
>>>>> base-apt (like it's done in this series).
>>>>>
>>>>>     - Manually parse 'debian/control'. But in this case base-apt should
>>>>> depend on *.bb:do_unpack. Due to bitbake is 'statically configured'
>>>>> tool, base-apt should know the list of all the recipes, what is also
>>>>> impossible. In previous series I've tried to implement this by
>>>>> defining a list of images in some global variable.
>>>> "Manuall parse" is a NoGo, just look at the multiple broken versions of
>>>> perl code guessing the build-deps ... emulating something that debian
>>>> itself just gets right.
>>>>   
>>>>> 2. Question how to get the list of build and runtime dependencies is
>>>>> still open, I've already asked about this in previous mail. So, the
>>>>> goal is to have the full list of packages that should be fetched.
>>>>> 'debian/control' contains very specific format, so I can't feed this
>>>>> line to multistrap/apt-get as it is. I see the following solutions:
>>>>>     - Implement custom tool to do this (apt/dpkg uses standard perl
>>>>> libs which contains all this stuff).
>>>>>     - Manually parse 'debian/control'
>>>>>     - Drop the requirement about single upstream fetch.
>>>> And again the infamous "manual parse" and even a "custom tool" ... No.
>>>>
>>>> The repo should come out of what the multiple "multistraps" and "apt-get
>>>> installs" actually fetch when they are allowed to fetch from the
>>>> outside. If you refer to that with "Drop the requirement about ..." i
>>>> fully agree.
>>> I agree with Henning's view. We should exploit Debian for building the
>>> repo and only use bitbake recipes to control and direct Debian tools,
>>> definitely not reimplement them.
>>>
>>> Having one build run that both generates outputs and fills that repos
>>> which would then be used on succeeding runs would be ok from user
>>> perspective. We do not necessarily need that strict ordering of
>>> fetch-before-use in the first build.
>>>
>>> Jan
>>>
>> I do also agree with Henning's view.
>>
>>
>> It would be better not struggling with those complexity within Isar.
>> Maybe I'm a bit late. But is reprepro really the best solution for
>> implementing the cache?
>> Let's assume we implement it like Henning mentioned already. We let Isar
>> generate its image and add all required packages to the local repository.
>>
>>
>> Then I see a big problem when adding new recipes to Isar, which will extend
>> the local apt repository with maybe new versions of a dependent package.
>> You never know that before.
>> In this case the new dependencies can trigger a bunch replacement of the
>> old packages, and the old stable state of local repo is gone.
>> The solution may be to create snapshots of the current local repository,
>> but reprepro is IMHO therefore not the best solution:
>> https://serverfault.com/questions/489504/using-snapshots-with-reprepro-to-enable-rollbacks
>>
>> I think "aptly" would be much better here, since aptly gives you the
>> possibility to create partial snapshots of new package versions.
>> It also comes along with snapshot merging and lot of things Isar
>> currently doesn't need, but maybe will sometime in the future.
>>
>>
>> I imagine the apt-cache as a real cache. Means this cache should be able
>> to be the interface between Isar and downloading the packages.
>> So the cache should download the package (and dependencies) into the
>> cache and Isar installs those packages from the cache instead from the
>> remote repository.
>> I have not tested that with aptly, but the link looks like it can
>> achieve that:
>> https://www.aptly.info/tutorial/pull/
>> When aptly is capable of downloading single packages (and dependencies)
>> from a remote repo, then i strongly recommend using this tool instead
>> of reprepro.
>>
> I also started to look into aptly, which I had on my notes for a while.
> One reason is that it could act as a manual workaround for the missing
> version-freezing support in Isar (pull into aptly repo, take a snapshot,
> and then build from it with Isar).
>
> Another is that we also need some tooling to manage mirrors and
> snapshots on the long run, and reprepro, TBHO, smells strange. On first
> glance, it looks to me like the multistrap of the repo management tools:
> once a cool thing, but now unmaintained.
>
> Did you dig further into aptly? I'm currently wondering how it
> internally manages snapshots, if it does proper deduplication and how
> much of that can be preserved when publishing for consumption by, say,
> Isar...
>
> Jan
>
Yes,

Until now I have implemented two approaches.

So I will offer some of my results tomorrow.

What I first did, was to use aptly as a complete mirror. Aptly creates a 
mirror and each package, which is requested by Isar will be added to the 
mirror and then be downloaded from there.

Unfortunately that was not as easy as expected, since aptly cannot 
download packages directly. Aptly was originally developed for creating 
complete debian mirrors.

It has now the "-filter" option, which can be used for narrowing the 
packages (and also dependecies) to be mirrored.

So for each additional package I extended the filter and updated the 
mirror afterwards, but that architecture appeared to suffer under less 
performance, since each extension of the filter

and subsequent update caused a download of debians packages.gz file (~ 
10MB).


My current approach is to create only a local repo after the first Isar 
build suceeded. This avoids complexity for now, since Isar and all 
debian tools are resolving dependencies as usual, when

running against upstream repos.

After the first build I create a snapshot of the local repo (both is 
possible: creating snapshots of local repos and also mirrors).


I'm just investigating the deduplication topic and will give you response.


Benni

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Jan Kiszka]

On 2018-01-24 21:53, Benedikt Niedermayr wrote:
> Am 24.01.2018 um 19:48 schrieb Jan Kiszka:
>> I also started to look into aptly, which I had on my notes for a while.
>> One reason is that it could act as a manual workaround for the missing
>> version-freezing support in Isar (pull into aptly repo, take a snapshot,
>> and then build from it with Isar).
>>
>> Another is that we also need some tooling to manage mirrors and
>> snapshots on the long run, and reprepro, TBHO, smells strange. On first
>> glance, it looks to me like the multistrap of the repo management tools:
>> once a cool thing, but now unmaintained.
>>
>> Did you dig further into aptly? I'm currently wondering how it
>> internally manages snapshots, if it does proper deduplication and how
>> much of that can be preserved when publishing for consumption by, say,
>> Isar...
>>
>> Jan
>>
> Yes,
> 
> Until now I have implemented two approaches.
> 
> So I will offer some of my results tomorrow.
> 
> What I first did, was to use aptly as a complete mirror. Aptly creates a
> mirror and each package, which is requested by Isar will be added to the
> mirror and then be downloaded from there.
> 
> Unfortunately that was not as easy as expected, since aptly cannot
> download packages directly. Aptly was originally developed for creating
> complete debian mirrors.
> 
> It has now the "-filter" option, which can be used for narrowing the
> packages (and also dependecies) to be mirrored.
> 
> So for each additional package I extended the filter and updated the
> mirror afterwards, but that architecture appeared to suffer under less
> performance, since each extension of the filter
> 
> and subsequent update caused a download of debians packages.gz file (~
> 10MB).
> 

You went down the same path I was exploring the past hours. But I
dropped as well, also because of such traps:

https://groups.google.com/forum/#!searchin/aptly-discuss/partial$20mirroring%7Csort:date/aptly-discuss/SCMrc0w5Mq0/up4RpnMLBgAJ

> 
> My current approach is to create only a local repo after the first Isar
> build suceeded. This avoids complexity for now, since Isar and all
> debian tools are resolving dependencies as usual, when
> 
> running against upstream repos.

Would be interested to see the changes you did for that.

> 
> After the first build I create a snapshot of the local repo (both is
> possible: creating snapshots of local repos and also mirrors).
> 
> 
> I'm just investigating the deduplication topic and will give you response.
> 

Deduplication seems to be done by aptly simply by creating hardlinks.
That even works nicely when publishing to a local directory.

Thinking the aptly path further: I quickly started to dream of a lazy
mirror. What if we could teach it to create a full mirror but only fetch
packages when someone requests them?

That means we would first need a live publication services using the
built-in webserver or something like inotify or even fuse for file-based
publication. Then we could track accesses and pull from upstream on
demand. After the first build, we would snapshot the result and use that
from then on. The impact on Isar would be minimal, no separate
dependency tracking, no Debian logic duplication, simple user interface.

That's very similar to what Claudius initially proposed, but we would be
building on top of an existing, mature repo management tool.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [RFC v2][PATCH 2/3] build-rep: Add helper class

[Benedikt Niedermayr]

[-- Attachment #1: Type: text/plain, Size: 5959 bytes --]

Am 24.01.2018 um 22:31 schrieb Jan Kiszka:
> On 2018-01-24 21:53, Benedikt Niedermayr wrote:
>> Am 24.01.2018 um 19:48 schrieb Jan Kiszka:
>>> I also started to look into aptly, which I had on my notes for a while.
>>> One reason is that it could act as a manual workaround for the missing
>>> version-freezing support in Isar (pull into aptly repo, take a snapshot,
>>> and then build from it with Isar).
>>>
>>> Another is that we also need some tooling to manage mirrors and
>>> snapshots on the long run, and reprepro, TBHO, smells strange. On first
>>> glance, it looks to me like the multistrap of the repo management tools:
>>> once a cool thing, but now unmaintained.
>>>
>>> Did you dig further into aptly? I'm currently wondering how it
>>> internally manages snapshots, if it does proper deduplication and how
>>> much of that can be preserved when publishing for consumption by, say,
>>> Isar...
>>>
>>> Jan
>>>
>> Yes,
>>
>> Until now I have implemented two approaches.
>>
>> So I will offer some of my results tomorrow.
>>
>> What I first did, was to use aptly as a complete mirror. Aptly creates a
>> mirror and each package, which is requested by Isar will be added to the
>> mirror and then be downloaded from there.
>>
>> Unfortunately that was not as easy as expected, since aptly cannot
>> download packages directly. Aptly was originally developed for creating
>> complete debian mirrors.
>>
>> It has now the "-filter" option, which can be used for narrowing the
>> packages (and also dependecies) to be mirrored.
>>
>> So for each additional package I extended the filter and updated the
>> mirror afterwards, but that architecture appeared to suffer under less
>> performance, since each extension of the filter
>>
>> and subsequent update caused a download of debians packages.gz file (~
>> 10MB).
>>
> You went down the same path I was exploring the past hours. But I
> dropped as well, also because of such traps:
>
> https://groups.google.com/forum/#!searchin/aptly-discuss/partial$20mirroring%7Csort:date/aptly-discuss/SCMrc0w5Mq0/up4RpnMLBgAJ
>
>> My current approach is to create only a local repo after the first Isar
>> build suceeded. This avoids complexity for now, since Isar and all
>> debian tools are resolving dependencies as usual, when
>>
>> running against upstream repos.
> Would be interested to see the changes you did for that.
>
>> After the first build I create a snapshot of the local repo (both is
>> possible: creating snapshots of local repos and also mirrors).
>>
>>
>> I'm just investigating the deduplication topic and will give you response.
>>
> Deduplication seems to be done by aptly simply by creating hardlinks.
> That even works nicely when publishing to a local directory.
>
> Thinking the aptly path further: I quickly started to dream of a lazy
> mirror. What if we could teach it to create a full mirror but only fetch
> packages when someone requests them?
>
> That means we would first need a live publication services using the
> built-in webserver or something like inotify or even fuse for file-based
> publication. Then we could track accesses and pull from upstream on
> demand. After the first build, we would snapshot the result and use that
> from then on. The impact on Isar would be minimal, no separate
> dependency tracking, no Debian logic duplication, simple user interface.
>
> That's very similar to what Claudius initially proposed, but we would be
> building on top of an existing, mature repo management tool.
>
> Jan
>


Hi,


> Would be interested to see the changes you did for that.
I can only give you my current implementation of my apt-cache.bbclass 
and dpkg-cross.bbclass.

The apt-cache.bbclass contains a simple shell based API for accessing 
the aptly functionality.

I will push that results to my next branch at my isar fork tomorrow.


> Deduplication seems to be done by aptly simply by creating hardlinks.
> That even works nicely when publishing to a local directory.
>
> Thinking the aptly path further: I quickly started to dream of a lazy
> mirror. What if we could teach it to create a full mirror but only fetch
> packages when someone requests them?
>
> That means we would first need a live publication services using the
> built-in webserver or something like inotify or even fuse for file-based
> publication. Then we could track accesses and pull from upstream on
> demand. After the first build, we would snapshot the result and use that
> from then on. The impact on Isar would be minimal, no separate
> dependency tracking, no Debian logic duplication, simple user interface.
>
> That's very similar to what Claudius initially proposed, but we would be
> building on top of an existing, mature repo management tool.
That sounds very nice. I don't know how to find good solution/design for 
that. But maybe a good brain wave might change that :-)


I also thinking about combining buildfragments (debian packages) with 
Bitbake's "setscene" mechanism, in order to create a sstate cache 
functionality, like yocto does.

I also don't know how to achieve that goal, since there is very less 
(means none) documentation about setscene, and the only point to start 
from there is the sstate implementation

of yocto itself. So first of all I have to understand the masses of code 
related to the sstate cache.

It is a big goal but it is also very nice feature and may solve a lot of 
problems related to mirroring or snapshoting a repository, since then a 
direct relationship between task hashes and

build fragments (debian packages) would be given.

If any of the task changes in a recipe, a new/old build fragment would 
be the result. And checking out any commit of a meta-layer would relate 
to matching build fragments.

This would be very helpful when checking out to a sooner commit of Isar 
(or any other layer).

A relationship between your current git repository state and related 
mirror could then be guaranteed.



Benni









[-- Attachment #2: apt-cache.bbclass --]
[-- Type: text/plain, Size: 5045 bytes --]

# This software is a part of ISAR.
# Copyright (C) 2017-2018 Mixed-Mode GmbH
#
# This class implements common functionality for the apt cache.
#
# Todo:
# 1. When a function of this API return an error, bitbake should output the last function produced the error!
#
# 2. Add functionality of creating snapshots of local isar repo.
#
#
CFG_FILE = "${CACHE_CONF_DIR}/${CACHE_CFG_FILE}"
MIRROR = "${DISTRO_SUITE}-mirror"
SNAPSHOT_BASENAME="snapshot"
ISAR_REPO_LOCAL = "isar-repo-local"
ISAR_FIRST_BUILD_DONE = "${TMPDIR}/first_build_done"

# Reproducible build is not default
ISAR_REPRODUCIBLE_BUILD ?= "1"


# Rename a snapshot
# $1: Old name.
# $2: New name.
_rename_snapshot() {
	aptly -config=${CFG_FILE} snapshot rename $1 $2
}


# Create a new repository database for Isar generated packages.
# $1: Repository database name
_repo_db_create() {
    aptly -config=${CFG_FILE} \
          repo create \
          -component="main" \
          -distribution="${DISTRO_SUITE}" \
          $1
}


# Drop a created repository database silenty.
_repo_db_drop() {
    aptly -config=${CFG_FILE} \
          repo drop \
          $1 || bbwarn "drop db failed"

}


# Add packages to the local repository database.
# $1: Repository database name.
# $*: Packages to add.
_repo_db_add_package() {
    repo=$1
    shift
    aptly -config=${CFG_FILE} repo add -force-replace $repo $*
}


# Delete packages from the repository database.
# $1: Repository database name.
# $*: Packages to delete.
_repo_db_delete_package() {
    #
    # Todo: Remove from pool? But maybe this is required for snapshots?
    # We have build fragments saved at the pool! That is good.
    #
    # After 3 rebuilds:
    #
    #./public/isar/pool/main/s/swupdate/swupdate_1.0_armhf.deb
    #./pool/df/88/702fa609fbd90b735d7cdb6dfeef_swupdate_1.0_armhf.deb
    #./pool/f0/6e/21d09a07366d7f9f7f6690ea7656_swupdate_1.0_armhf.deb
    #./pool/bd/c3/974c8903d41666d23239a8330627_swupdate_1.0_armhf.deb

    repo=$1
    shift
    aptly -config=${CFG_FILE} repo remove $repo $*
}


# Publish repository consumed by apt. This function can only be called
# for repositories
# $1: Repository database name
# $2: Path Prefix
_repo_db_publish(){
    aptly -config=${CFG_FILE} \
          publish repo \
          -architectures="${DEB_HOST_ARCH},${DISTRO_ARCH}" \
          $1 \
          $2

}


# Drop an already published repository.
# $1: Path prefix
_repo_db_publish_drop() {
    aptly -config=${CFG_FILE} \
          publish drop \
          ${DISTRO_SUITE} \
          $1 || bbwarn "drop published failed"
}


# Switch already published repository to new repository. This
# will change the content of the published repository. This function
# can only be called on published repos.
# $1: Prefix.
_repo_db_published_update() {
    aptly -config=${CFG_FILE} \
          publish update \
          -force-overwrite \
          ${DISTRO_SUITE} $1
}


# Add packages to the isar cache.
# $1: Repository database name.
# $2: Repository prefix
# $*: Packages to add.
cache_add_package() {
    repo=$1
    prefix=$2
    shift 2
    _repo_db_add_package $repo $*
    _repo_db_published_update $prefix
}


# Delete a package from the apt cache.
# This function is used to be called within do_clean(all) task.
# $1: Repository database name.
# $2: Repository prefix
# $*: Packages to delete.
cache_delete_package() {
    repo=$1
    prefix=$2
    shift 2
    _repo_db_delete_package $repo $*
    _repo_db_published_update $prefix
}


cache_create_snapshot() {
    aptly -config=${CFG_FILE} \
          snapshot create ${SNAPSHOT_BASENAME}_${DATETIME} from repo ${ISAR_REPO_LOCAL}
}


cache_load_snapshot() {
  "${FUNCNAME[1]} not implemented."
}

# Setup the apt cache, only if no cache is available, yet.
do_cache_setup() {
    set -E
    _cleanup() {
        ret=$?
        bbwarn "Error in apt-cache.bblcass: ${BASH_SOURCE[0]}: (Line ${BASH_LINENO[0]}, Func:${FUNCNAME[1]})"
        (exit $ret) || bb_exit_handler
    }
    trap '_cleanup' ERR

    # Very first build of isar. Setting up the cache.
    if [ ! -e "${CFG_FILE}" ]; then
        bbplain "Creating local apt cache for first build."
        sed -e 's|##CACHE_DIR##|${CACHE_DIR}|' \
            ${FILESDIR}/${CACHE_CFG_FILE}.in > ${CFG_FILE}

        _repo_db_create ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX}
        _repo_db_publish ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX}
    fi


    if [ -e "${ISAR_FIRST_BUILD_DONE}" ] && [ "${REPRODUCIBLE_BUILD_ENABLED}" == "1" ]; then
        bbplain "Reusing last apt cache snapshot: $(aptly --config=${CFG_FILE} --raw -sort=time snapshot list | tail -n 1)"
        return
    fi

    #bbwarn "Republishing repo of last build"
    #_repo_db_publish_drop ${ISAR_CACHE_LOCAL_PREFIX}
    #_repo_db_drop ${ISAR_REPO_LOCAL}
    #_repo_db_create ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX}
    #_repo_db_publish ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX}

}
addtask do_cache_setup before do_rootfs
do_cache_setup[dirs] += "${CACHE_CONF_DIR}"
do_cache_setup[lockfiles] = "${DPKG_LOCK}"

[-- Attachment #3: dpkg-cross.bbclass --]
[-- Type: text/plain, Size: 2089 bytes --]

# This software is a part of ISAR.
# Copyright (C) 2017 Mixed Mode GmbH

# Class implementing common functionalities for running cross builds
# for debianized packages.
# Do not directly inherit this class.
# Instead inherit from cross.bbclass, which is done by setting the BBCLASSEXTEND = "cross" within your recipe.

inherit fetch patch apt-cache

DEPENDS += "cross-buildchroot"
do_unpack[deptask] = "do_install"
do_install[rdeptask] = "do_install"

CHROOT_DIR = "${CROSS_BUILDCHROOT_DIR}"
BUILDROOT = "${CHROOT_DIR}/${PP}"
EXTRACTDIR = "${BUILDROOT}"
S = "${EXTRACTDIR}/${SRC_DIR}"
B = "${BUILDROOT}/${BUILD_DIR}"


python () {
    s = d.getVar('S', True).rstrip('/')
    extract_dir = d.getVar('EXTRACTDIR', True).rstrip('/')

    if s == extract_dir:
        bb.fatal('\nS equals EXTRACTDIR. Maybe SRC_DIR variable was not set.')
}


OPTS = "--tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y ${APT_EXTRA_OPTS}'"
do_install_depends() {
    cd ${PPS}
    apt-get update
    mk-build-deps ${OPTS} -i -r debian/control
}
addtask do_install_depends after do_patch before do_build
do_install_depends[lockfiles] = "${DPKG_LOCK}"
do_install_depends[stamp-extra-info] = "${MACHINE}.chroot"
do_install_depends[chroot] = "1"
do_install_depends[id] = "${CROSS_BUILDCHROOT_ID}"


# Build package from sources
do_build() {
    cd ${PPS}
    dpkg-buildpackage ${DEB_SIGN} -pgpg -sn --host-arch=${DEB_ARCH} -Z${DEB_COMPRESSION}
}
do_build[stamp-extra-info] = "${MACHINE}.chroot"
do_build[chroot] = "1"
do_build[id] = "${CROSS_BUILDCHROOT_ID}"


do_install() {
    cache_add_package ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX} ${BUILDROOT}/*.deb
}
addtask do_install after do_build
do_install[lockfiles] = "${DPKG_LOCK}"
do_install[stamp-extra-info] = "${DISTRO}"
do_install[dirs] += "${DEPLOY_DIR_IMAGE}"


do_clean_cache_pkg() {
    pkgs="$(ls ${BUILDROOT}/*.deb)"
    for pkg in $pkgs ; do
        cache_delete_package ${ISAR_REPO_LOCAL} ${ISAR_CACHE_LOCAL_PREFIX} "$(basename $pkg | sed 's/_.*.deb//')"
    done
}
addtask do_clean_cache_pkg before do_clean do_cleanall