Re: [PATCH] meta/recipes-support: Mount overlay-fs for /etc

[Jan Kiszka <jan.kiszka@siemens.com>]

On 22.04.20 10:24, Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> Add an overlay fs[1] mount for /etc. This overlay allows an image update of
> root file-system without overwriting the user settings in /etc. The overlay
> mounts during the systemd target `local-fs-pre.target.` Due to this target
> the by systemd generated fsck service is not a dependency for the overlay.
> 
> [1]: https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
>   doc/user_manual.md                            | 26 ++++++
>   meta-isar/conf/machine/qemuamd64.conf         |  4 +-
>   .../etc-overlay/etc-overlay_0.1.bb            | 39 +++++++++
>   .../etc-overlay/files/etc-hostname.service    | 14 ++++
>   .../etc-overlay/files/etc.mount               | 15 ++++
>   .../files/overlay-parse-etc.service           | 14 ++++
>   .../etc-overlay/files/ovl.mount.tmpl          | 14 ++++
>   .../etc-overlay/files/postinst                | 32 +++++++
>   meta/recipes-support/etc-overlay/files/postrm | 29 +++++++
>   scripts/lib/wic/canned-wks/etc-overlay.inc    |  5 ++
>   scripts/lib/wic/plugins/source/etc-overlay.py | 84 +++++++++++++++++++
>   11 files changed, 275 insertions(+), 1 deletion(-)
>   create mode 100644 meta/recipes-support/etc-overlay/etc-overlay_0.1.bb
>   create mode 100644 meta/recipes-support/etc-overlay/files/etc-hostname.service
>   create mode 100644 meta/recipes-support/etc-overlay/files/etc.mount
>   create mode 100644 meta/recipes-support/etc-overlay/files/overlay-parse-etc.service
>   create mode 100644 meta/recipes-support/etc-overlay/files/ovl.mount.tmpl
>   create mode 100755 meta/recipes-support/etc-overlay/files/postinst
>   create mode 100644 meta/recipes-support/etc-overlay/files/postrm
>   create mode 100644 scripts/lib/wic/canned-wks/etc-overlay.inc
>   create mode 100644 scripts/lib/wic/plugins/source/etc-overlay.py
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index d13a74e..5ec7003 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -901,3 +901,29 @@ And build the corresponding image target:
>   ```
>   bitbake mc:qemuarm64-buster:isar-image-base
>   ```
> +## Mount overlay for /etc
> +
> +### Motivation
> +
> +If a software update solution updates the complete image, the content of `/etc`
> +is also set to new image content. In this case, the user needs to reconfigure
> +her system.
> +
> +### Approach/Solution
> +
> +A possible solution is to create an additional partition, which superimposes
> +`/etc` with an [overlay file-system](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt).
> +
> +### Example
> +
> +Add the following lines to local.conf:
> +```
> +IMAGE_TYPE = "wic-img"
> +WKS_FILE = "sdimage-efi-overlay"
> +IMAGE_INSTALL += "etc-overlay"
> +```
> +And build the corresponding image target:
> +
> +```
> +bitbake mc:qemuamd64-buster:isar-image-base
> +```
> diff --git a/meta-isar/conf/machine/qemuamd64.conf b/meta-isar/conf/machine/qemuamd64.conf
> index 028b738..d429d96 100644
> --- a/meta-isar/conf/machine/qemuamd64.conf
> +++ b/meta-isar/conf/machine/qemuamd64.conf
> @@ -6,11 +6,13 @@ DISTRO_ARCH ?= "amd64"
>   KERNEL_NAME ?= "amd64"
>   
>   IMAGE_TYPE ?= "wic-img"
> -WKS_FILE ?= "sdimage-efi"
> +WKS_FILE ?= "sdimage-efi-overlay"
>   IMAGER_INSTALL += "${GRUB_BOOTLOADER_INSTALL}"
>   
>   IMAGE_INSTALL += "sshd-regen-keys"
>   
> +IMAGE_INSTALL += "etc-overlay"
> +
>   QEMU_ARCH ?= "x86_64"
>   QEMU_MACHINE ?= "q35"
>   QEMU_CPU ?= ""
> diff --git a/meta/recipes-support/etc-overlay/etc-overlay_0.1.bb b/meta/recipes-support/etc-overlay/etc-overlay_0.1.bb
> new file mode 100644
> index 0000000..cbed71d
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/etc-overlay_0.1.bb
> @@ -0,0 +1,39 @@
> +# Create a overlay for /etc to freeze a default configuration
> +#
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2020
> +#
> +# SPDX-License-Identifier: MIT
> +
> +
> +DESCRIPTION = "overlay systemd-mount"
> +
> +DEBIAN_DEPENDS = "systemd"
> +
> +SRC_URI = "file://postinst \
> +           file://postrm \
> +           file://etc.mount \
> +           file://ovl.mount.tmpl \
> +           file://overlay-parse-etc.service \
> +           file://etc-hostname.service"
> +
> +FS_COMMIT_INTERVAL ?= "20"
> +
> +TEMPLATE_VARS  += "FS_COMMIT_INTERVAL"
> +TEMPLATE_FILES += "ovl.mount.tmpl"
> +
> +inherit dpkg-raw
> +
> +do_install() {
> +    install -m 0755 -d ${D}/ovl
> +    touch ${D}/ovl/.keep
> +
> +    TARGET=${D}/lib/systemd/system
> +    install -m 0755 -d ${TARGET}
> +    install -m 0644 ${WORKDIR}/etc.mount ${TARGET}/etc.mount
> +    install -m 0644 ${WORKDIR}/ovl.mount ${TARGET}/ovl.mount
> +    install -m 0644 ${WORKDIR}/overlay-parse-etc.service  ${TARGET}/overlay-parse-etc.service
> +    install -m 0644 ${WORKDIR}/etc-hostname.service ${TARGET}/etc-hostname.service
> +}
> +
> +addtask do_install after do_transform_template
> diff --git a/meta/recipes-support/etc-overlay/files/etc-hostname.service b/meta/recipes-support/etc-overlay/files/etc-hostname.service
> new file mode 100644
> index 0000000..2306b9f
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/etc-hostname.service
> @@ -0,0 +1,14 @@
> +[Unit]
> +Description=set hostname /etc overlay-aware
> +Before=network-pre.target
> +Wants=network-pre.target
> +Requires=etc.mount
> +After=etc.mount
> +
> +[Service]
> +Type=oneshot
> +RemainAfterExit=yes
> +ExecStart=/bin/hostname --boot --file /etc/hostname
> +
> +[Install]
> +WantedBy=basic.target
> diff --git a/meta/recipes-support/etc-overlay/files/etc.mount b/meta/recipes-support/etc-overlay/files/etc.mount
> new file mode 100644
> index 0000000..59ee0d7
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/etc.mount
> @@ -0,0 +1,15 @@
> +[Unit]
> +Description=Overlay-mount /etc
> +DefaultDependencies=no
> +Before=local-fs-pre.target
> +Requires=ovl.mount
> +After=ovl.mount
> +
> +[Mount]
> +What=overlay
> +Where=/etc
> +Type=overlay
> +Options=noauto,x-systemd.automount,lowerdir=/etc,upperdir=/ovl/etc,workdir=/ovl/.atomic
> +
> +[Install]
> +WantedBy=local-fs-pre.target
> diff --git a/meta/recipes-support/etc-overlay/files/overlay-parse-etc.service b/meta/recipes-support/etc-overlay/files/overlay-parse-etc.service
> new file mode 100644
> index 0000000..eb5d422
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/overlay-parse-etc.service
> @@ -0,0 +1,14 @@
> +[Unit]
> +Description=Reload Configuration from the etc overlay
> +DefaultDependencies=no
> +Requires=etc.mount
> +After=etc.mount
> +Before=local-fs-pre.target
> +
> +[Service]
> +Type=oneshot
> +RemainAfterExit=yes
> +ExecStartPre=!/bin/systemctl daemon-reload
> +ExecStart=!/bin/systemctl --no-block isolate multi-user.target
> +[Install]
> +WantedBy=local-fs-pre.target
> diff --git a/meta/recipes-support/etc-overlay/files/ovl.mount.tmpl b/meta/recipes-support/etc-overlay/files/ovl.mount.tmpl
> new file mode 100644
> index 0000000..3593c55
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/ovl.mount.tmpl
> @@ -0,0 +1,14 @@
> +[Unit]
> +Description=Mount /etc overlay backing store
> +DefaultDependencies=no
> +Before=local-fs-pre.target
> +Before=etc.mount
> +
> +[Mount]
> +What=/dev/disk/by-partlabel/etcovl
> +Where=/ovl
> +Type=ext4
> +Options=noatime,nodiratime,data=journal,commit=${FS_COMMIT_INTERVAL},nodelalloc
> +
> +[Install]
> +WantedBy=local-fs-pre.target
> diff --git a/meta/recipes-support/etc-overlay/files/postinst b/meta/recipes-support/etc-overlay/files/postinst
> new file mode 100755
> index 0000000..7a10d68
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/postinst
> @@ -0,0 +1,32 @@
> +#!/bin/sh
> +# postinst script for etc-overlay
> +#
> +# see: dh_installdeb(1)
> +
> +set -e
> +
> +case "$1" in
> +    configure)
> +        deb-systemd-helper enable etc.mount  || true
> +        deb-systemd-helper enable ovl.mount  || true
> +        deb-systemd-helper enable overlay-parse-etc.service || true
> +        deb-systemd-helper enable systemd-remount-fs.service || true
> +        deb-systemd-helper enable etc-hostname.service || true
> +    ;;
> +
> +    abort-upgrade|abort-remove|abort-deconfigure)
> +    ;;
> +
> +    *)
> +        echo "postinst called with unknown argument \`$1'" >&2
> +        exit 1
> +        ;;
> +esac
> +
> +# dh_installdeb will replace this with shell code automatically
> +# generated by other debhelper scripts.
> +
> +#DEBHELPER#
> +
> +exit 0
> +
> diff --git a/meta/recipes-support/etc-overlay/files/postrm b/meta/recipes-support/etc-overlay/files/postrm
> new file mode 100644
> index 0000000..7a3defb
> --- /dev/null
> +++ b/meta/recipes-support/etc-overlay/files/postrm
> @@ -0,0 +1,29 @@
> +#!/bin/sh
> +# postrm script for etc-overlay
> +#
> +# see: dh_installdeb(1)
> +
> +set -e
> +
> +
> +case "$1" in
> +    purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
> +        deb-systemd-helper disable etc.mount  || true
> +        deb-systemd-helper disable ovl.mount  || true
> +        deb-systemd-helper disable overlay-parse-etc.service || true
> +        deb-systemd-helper disable systemd-remount-fs.service || true
> +        deb-systemd-helper disable etc-hostname.service || true
> +     ;;
> +
> +    *)
> +        echo "postrm called with unknown argument \`$1'" >&2
> +        exit 1
> +    ;;
> +esac
> +
> +# dh_installdeb will replace this with shell code automatically
> +# generated by other debhelper scripts.
> +
> +#DEBHELPER#
> +
> +exit 0
> diff --git a/scripts/lib/wic/canned-wks/etc-overlay.inc b/scripts/lib/wic/canned-wks/etc-overlay.inc
> new file mode 100644
> index 0000000..7b40854
> --- /dev/null
> +++ b/scripts/lib/wic/canned-wks/etc-overlay.inc
> @@ -0,0 +1,5 @@
> +# add a overlay partition to the image
> +
> +# overlay partition
> +part --source etc-overlay --ondisk sda --size 100M --extra-space 128M --overhead-factor 1 --label etcovl --align 1024 --fstype=ext4
> +

Where is this used/tested? I think you should create a test/demo target 
that uses a read-only rootfs and such an overlay partition via the include.

Jan

> diff --git a/scripts/lib/wic/plugins/source/etc-overlay.py b/scripts/lib/wic/plugins/source/etc-overlay.py
> new file mode 100644
> index 0000000..55189a2
> --- /dev/null
> +++ b/scripts/lib/wic/plugins/source/etc-overlay.py
> @@ -0,0 +1,84 @@
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +#
> +# Copyright (c) 2014, Intel Corporation.
> +# Copyright (c) 2018, Siemens AG.
> +# All rights reserved.
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +#
> +# DESCRIPTION
> +# This implements the 'etc-overlay' source plugin class for 'wic'
> +#
> +# AUTHORS
> +# Tom Zanussi <tom.zanussi (at] linux.intel.com>
> +# Andreas Reichel <andreas.reichel.ext (at] siemens.com>
> +# Quirin Gylstorff <quirin.gylstorff [at] siemens.com>
> +
> +
> +import logging
> +
> +msger = logging.getLogger('wic')
> +
> +from wic.pluginbase import SourcePlugin
> +from wic.utils.misc import exec_cmd,BOOTDD_EXTRA_SPACE
> +
> +class EtcOverlayPlugin(SourcePlugin):
> +    """
> +    Create an overlay file system scheme for etc
> +    """
> +
> +    name = 'etc-overlay'
> +
> +    @classmethod
> +    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
> +                             oe_builddir, deploy_dir, kernel_dir,
> +                             rootfs_dir, native_sysroot):
> +
> +        part_rootfs_dir = "%s/disk/%s.%s" % (cr_workdir,
> +                                             part.label,
> +                                             part.lineno)
> +        create_dir_cmd = "install -d %s" % part_rootfs_dir
> +        exec_cmd(create_dir_cmd)
> +
> +        exec_cmd("install -m 0755 -d %s/etc" % part_rootfs_dir)
> +        exec_cmd("install -m 0755 -d %s/.atomic" % part_rootfs_dir)
> +
> +        blocks = 16
> +        extra_blocks = part.get_extra_block_count(blocks)
> +        if extra_blocks < BOOTDD_EXTRA_SPACE:
> +            extra_blocks = BOOTDD_EXTRA_SPACE
> +        blocks += extra_blocks
> +        blocks = blocks + (16 - (blocks % 16))
> +
> +        msger.debug("Added %d extra blocks to %s to get to %d total blocks",
> +                    extra_blocks, part.mountpoint, blocks)
> +
> +        # ext4 image, created by mkfs.ext4
> +        etcovlimg = "%s/%s.%s.img" % (cr_workdir, part.label, part.lineno)
> +        partfs_cmd = "dd if=/dev/zero of=%s bs=512 count=%d" % (etcovlimg,
> +                                                                blocks)
> +        exec_cmd(partfs_cmd)
> +
> +        partfs_cmd = "mkfs.ext4 %s -d %s" % (etcovlimg, part_rootfs_dir)
> +        exec_cmd(partfs_cmd)
> +
> +        chmod_cmd = "chmod 644 %s" % etcovlimg
> +        exec_cmd(chmod_cmd)
> +
> +        du_cmd = "du -Lbks %s" % etcovlimg
> +        etcovlimg_size = int(exec_cmd(du_cmd).split()[0])
> +
> +        part.size = etcovlimg_size
> +        part.source_file = etcovlimg
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux