[PATCH v3] debootstrap: Add fallback apt cache support

[PATCH v3] debootstrap: Add fallback apt cache support

[Alexander Smirnov]

Add apt config file to switch caching mechanism from MMap to array-based.
This needed to provide possibility to build fresh Isar suites on old hosts.
For example, building Isar stretch on jessie host fails with the following
error:

  Hit:1 http://security.debian.org stretch/updates InRelease
  Ign:2 http://ftp.de.debian.org/debian stretch InRelease
  Hit:3 http://ftp.de.debian.org/debian stretch-updates InRelease
  Hit:4 http://ftp.de.debian.org/debian stretch Release
  E: Dynamic MMap ran out of room. Please increase the size of APT::Cache-Start. Current value: 25165824. (man 5 apt.conf)
  qemu: uncaught target signal 11 (Segmentation fault) - core dumped
  Segmentation fault

Now Isar performs the following:
1. Checking if user attempts to build stretch on old kernels
2. Temporarly installing this apt config file.
3. Removing this file after image rootfs creation

So this file will not go to target rootfs.

Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
---
 meta-isar/recipes-core/images/isar-image-base.bb          |  1 +
 .../isar-bootstrap/files/isar-apt-fallback.conf           |  4 ++++
 meta/recipes-core/isar-bootstrap/isar-bootstrap.bb        | 15 ++++++++++++++-
 3 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf

diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
index 04c92ab..5ad6df3 100644
--- a/meta-isar/recipes-core/images/isar-image-base.bb
+++ b/meta-isar/recipes-core/images/isar-image-base.bb
@@ -45,4 +45,5 @@ do_rootfs() {
     sudo rmdir ${IMAGE_ROOTFS}/isar-apt
     sudo umount -l ${IMAGE_ROOTFS}/dev
     sudo umount -l ${IMAGE_ROOTFS}/proc
+    sudo rm -f "${IMAGE_ROOTFS}/etc/apt/apt.conf.d/55isar-fallback.conf"
 }
diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf b/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf
new file mode 100644
index 0000000..12ddbea
--- /dev/null
+++ b/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf
@@ -0,0 +1,4 @@
+# Switch apt caching mechanism from mmap to array-based.
+# This helps to build fresh Isar suites on old hosts.
+
+APT::Cache-Fallback "1";
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
index 97b6640..c9a3d98 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
@@ -10,7 +10,9 @@ Description = "Minimal debian root file system"
 LICENSE = "gpl-2.0"
 LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
 FILESPATH_prepend := "${THISDIR}/files:"
-SRC_URI = "file://isar-apt.conf"
+SRC_URI = " \
+    file://isar-apt.conf \
+    file://isar-apt-fallback.conf"
 PV = "1.0"
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
@@ -189,6 +191,11 @@ do_bootstrap() {
 }
 addtask bootstrap before do_build after do_generate_keyring
 
+def get_host_release():
+    import platform
+    rel = platform.release()
+    return rel
+
 do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_apt_config_install() {
     sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
@@ -201,6 +208,12 @@ do_apt_config_install() {
     sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
     sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
                           "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
+
+    if [ "${@get_distro_suite(d)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
+        sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
+                              "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
+    fi
+
 }
 addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
 
-- 
2.1.4

Re: [PATCH v3] debootstrap: Add fallback apt cache support

[Alexander Smirnov]

On 05/14/2018 04:58 PM, Alexander Smirnov wrote:
> Add apt config file to switch caching mechanism from MMap to array-based.
> This needed to provide possibility to build fresh Isar suites on old hosts.
> For example, building Isar stretch on jessie host fails with the following
> error:
> 
>    Hit:1 http://security.debian.org stretch/updates InRelease
>    Ign:2 http://ftp.de.debian.org/debian stretch InRelease
>    Hit:3 http://ftp.de.debian.org/debian stretch-updates InRelease
>    Hit:4 http://ftp.de.debian.org/debian stretch Release
>    E: Dynamic MMap ran out of room. Please increase the size of APT::Cache-Start. Current value: 25165824. (man 5 apt.conf)
>    qemu: uncaught target signal 11 (Segmentation fault) - core dumped
>    Segmentation fault
> 
> Now Isar performs the following:
> 1. Checking if user attempts to build stretch on old kernels
> 2. Temporarly installing this apt config file.
> 3. Removing this file after image rootfs creation
> 
> So this file will not go to target rootfs.
> 
> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> ---
>   meta-isar/recipes-core/images/isar-image-base.bb          |  1 +
>   .../isar-bootstrap/files/isar-apt-fallback.conf           |  4 ++++
>   meta/recipes-core/isar-bootstrap/isar-bootstrap.bb        | 15 ++++++++++++++-
>   3 files changed, 19 insertions(+), 1 deletion(-)
>   create mode 100644 meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf
> 
> diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
> index 04c92ab..5ad6df3 100644
> --- a/meta-isar/recipes-core/images/isar-image-base.bb
> +++ b/meta-isar/recipes-core/images/isar-image-base.bb
> @@ -45,4 +45,5 @@ do_rootfs() {
>       sudo rmdir ${IMAGE_ROOTFS}/isar-apt
>       sudo umount -l ${IMAGE_ROOTFS}/dev
>       sudo umount -l ${IMAGE_ROOTFS}/proc
> +    sudo rm -f "${IMAGE_ROOTFS}/etc/apt/apt.conf.d/55isar-fallback.conf"
>   }
> diff --git a/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf b/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf
> new file mode 100644
> index 0000000..12ddbea
> --- /dev/null
> +++ b/meta/recipes-core/isar-bootstrap/files/isar-apt-fallback.conf
> @@ -0,0 +1,4 @@
> +# Switch apt caching mechanism from mmap to array-based.
> +# This helps to build fresh Isar suites on old hosts.
> +
> +APT::Cache-Fallback "1";
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> index 97b6640..c9a3d98 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.bb
> @@ -10,7 +10,9 @@ Description = "Minimal debian root file system"
>   LICENSE = "gpl-2.0"
>   LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
>   FILESPATH_prepend := "${THISDIR}/files:"
> -SRC_URI = "file://isar-apt.conf"
> +SRC_URI = " \
> +    file://isar-apt.conf \
> +    file://isar-apt-fallback.conf"
>   PV = "1.0"
>   
>   WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
> @@ -189,6 +191,11 @@ do_bootstrap() {
>   }
>   addtask bootstrap before do_build after do_generate_keyring
>   
> +def get_host_release():
> +    import platform
> +    rel = platform.release()
> +    return rel
> +
>   do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>   do_apt_config_install() {
>       sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
> @@ -201,6 +208,12 @@ do_apt_config_install() {
>       sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
>       sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
>                             "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
> +
> +    if [ "${@get_distro_suite(d)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
> +        sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
> +                              "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
> +    fi
> +

This trailing line will be fixed before applying, sorry.

Alex

>   }
>   addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
>   
>