[PATCH v3 2/5] container-loader: Introduce helper to load container images into local registry

["'Jan Kiszka' via isar-users" <isar-users@googlegroups.com>]

From: Jan Kiszka <jan.kiszka@siemens.com>

This allows to write dpkg-raw recipes which packages archived container
images and load them into a local docker or podman registry on boot. The
scenario behind this is to pre-fill local registries in a way that still
permits live updates during runtime.

The loader script only process images which are not yet available under
the same name and tag in the local registry. Also after loading, the
archived images stay on the local file system. This allows to perform
reloading in case the local registry should be emptied (e.g. reset to
factory state). To reduce the space those original images need, they are
compressed, by default with zstd.

Separate include files are available to cater the main container
engines, one for docker and one for podman.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 .../container-loader/container-loader.inc     | 101 ++++++++++++++++++
 .../container-loader/docker-loader.inc        |  10 ++
 .../files/container-loader.service.tmpl       |  12 +++
 .../files/container-loader.sh.tmpl            |  18 ++++
 .../container-loader/podman-loader.inc        |  10 ++
 5 files changed, 151 insertions(+)
 create mode 100644 meta/recipes-support/container-loader/container-loader.inc
 create mode 100644 meta/recipes-support/container-loader/docker-loader.inc
 create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl
 create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl
 create mode 100644 meta/recipes-support/container-loader/podman-loader.inc

diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes-support/container-loader/container-loader.inc
new file mode 100644
index 00000000..e97e829b
--- /dev/null
+++ b/meta/recipes-support/container-loader/container-loader.inc
@@ -0,0 +1,101 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+FILESPATH:append := ":${FILE_DIRNAME}/files"
+
+inherit dpkg-raw
+
+SRC_URI += " \
+    file://container-loader.service.tmpl \
+    file://container-loader.sh.tmpl"
+
+CONTAINER_COMPRESSION ?= "zst"
+CONTAINER_DELETE_AFTER_LOAD ?= "0"
+
+DEBIAN_DEPENDS += " \
+    ${CONTAINER_ENGINE_PACKAGES} \
+    ${@', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+       ', zstd' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+       ', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+       ''}"
+
+CONTAINER_COMPRESSOR_CMD = "${@ \
+    'gzip -f -9 -n --rsyncable' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+    'xz -f ${XZ_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+    'zstd -f --rm ${ZSTD_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+    ''}"
+
+CONTAINER_DECOMPRESSOR_CMD = "${@ \
+    'gzip -c -d -n' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+    'xz -c -d -T0' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+    'pzstd -c -d' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+    ''}"
+
+TEMPLATE_FILES += " \
+    container-loader.service.tmpl \
+    container-loader.sh.tmpl"
+TEMPLATE_VARS += " \
+    CONTAINER_ENGINE \
+    CONTAINER_DECOMPRESSOR_CMD \
+    CONTAINER_DELETE_AFTER_LOAD"
+
+do_install() {
+    install -m 755 ${WORKDIR}/container-loader.sh ${D}/usr/share/${BPN}
+}
+do_install[cleandirs] += " \
+    ${D}/usr/share/${BPN} \
+    ${D}/usr/share/${BPN}/images"
+
+python do_install_fetched_containers() {
+    import os
+
+    workdir = d.getVar('WORKDIR')
+    D = d.getVar('D')
+    BPN = d.getVar('BPN')
+
+    image_list = open(D + "/usr/share/" + BPN + "/image.list", "w")
+
+    src_uri = d.getVar('SRC_URI').split()
+    for uri in src_uri:
+        scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri)
+        if scheme != "docker":
+            continue
+
+        image_name = host + (path if path != "/" else "")
+        unpacked_image = workdir + "/" + image_name.replace('/', '.')
+        dest_dir = D + "/usr/share/" + BPN + "/images"
+        tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar"
+        docker_ref = ":" + parm["tag"] if "tag" in parm else "latest"
+
+        bb.utils.remove(tar_image)
+        cmd = f"skopeo copy dir:{unpacked_image} " \
+            f"docker-archive:{tar_image}:{image_name}{docker_ref}"
+        bb.note(f"running: {cmd}")
+        bb.process.run(cmd)
+
+        cmd = f"{d.getVar('CONTAINER_COMPRESSOR_CMD')} {tar_image}"
+        bb.note(f"running: {cmd}")
+        bb.process.run(cmd)
+
+        line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \
+            image_name + docker_ref
+        bb.note(f"adding '{line}' to image.list")
+        image_list.write(line + "\n")
+
+    image_list.close()
+}
+
+addtask install_fetched_containers after do_install before do_prepare_build
+
+do_prepare_build:append() {
+    install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${BPN}.service
+
+    # Do not compress the package, most of its payload is already, and trying
+    # nevertheless will only cost time without any gain.
+    cat <<EOF >> ${S}/debian/rules
+override_dh_builddeb:
+	dh_builddeb -- -Znone
+EOF
+}
diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes-support/container-loader/docker-loader.inc
new file mode 100644
index 00000000..b864c854
--- /dev/null
+++ b/meta/recipes-support/container-loader/docker-loader.inc
@@ -0,0 +1,10 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+require container-loader.inc
+
+CONTAINER_ENGINE = "docker"
+
+CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor"
diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl b/meta/recipes-support/container-loader/files/container-loader.service.tmpl
new file mode 100644
index 00000000..1638eaf2
--- /dev/null
+++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl
@@ -0,0 +1,12 @@
+[Unit]
+Description=Load archived container images on boot
+After=${CONTAINER_ENGINE}.service
+Requires=${CONTAINER_ENGINE}.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/share/${BPN}/container-loader.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl
new file mode 100755
index 00000000..b6abec92
--- /dev/null
+++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl
@@ -0,0 +1,18 @@
+#!/bin/sh
+#
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+set -eu
+
+while read -r image ref; do
+    if [ -e /usr/share/${BPN}/images/"$image" ] && \
+       [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then
+        ${CONTAINER_DECOMPRESSOR_CMD} /usr/share/${BPN}/images/"$image" | \
+            ${CONTAINER_ENGINE} load
+        if [ "${CONTAINER_DELETE_AFTER_LOAD}" = "1" ]; then
+            rm -f /usr/share/${BPN}/images/"$image"
+        fi
+    fi
+done < /usr/share/${BPN}/image.list
diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes-support/container-loader/podman-loader.inc
new file mode 100644
index 00000000..d2c9a12d
--- /dev/null
+++ b/meta/recipes-support/container-loader/podman-loader.inc
@@ -0,0 +1,10 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+require container-loader.inc
+
+CONTAINER_ENGINE = "podman"
+
+CONTAINER_ENGINE_PACKAGES ?= "podman"
-- 
2.43.0

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/isar-users/3b7b8dbdde7fa3a4184daa3f8d567e72c8b50d2d.1721139489.git.jan.kiszka%40siemens.com.