[PATCH] debianize: handle .triggers and .service files

[PATCH] debianize: handle .triggers and .service files

[Cedric Hombourger]

Some packages need to update/rebuild the initramfs and this is particularly
slow on Debian systems (which are not using more modern technologies such
as dracut or mkosi). Instead of having each package call update-initramfs,
use a trigger instead to have dpkg defer that call to the very end of the
transaction. Many packages also failed to rebuild the initramfs when the
package gets removed. Demonstrate use with initramfs-fsck-hook-ext4.

Other packages ship systemd units and usually do this using do_install()
and a postinst script. Such recipes also fail to handle the removal of
such packages. By naming the unit ${PN}.service and putting it under
debian/, debhelper will generate the necessary maintainer scripts
(preinst, postinst, prerm, postrm).

Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
---
 meta/classes/debianize.bbclass                            | 8 ++++++++
 ...d-on-first-boot_1.4.bb => expand-on-first-boot_1.5.bb} | 6 +-----
 meta/recipes-support/expand-on-first-boot/files/postinst  | 3 ---
 .../files/{postinst => initramfs-fsck-hook-ext4.triggers} | 5 ++---
 ...k-hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} | 2 +-
 5 files changed, 12 insertions(+), 12 deletions(-)
 rename meta/recipes-support/expand-on-first-boot/{expand-on-first-boot_1.4.bb => expand-on-first-boot_1.5.bb} (78%)
 delete mode 100644 meta/recipes-support/expand-on-first-boot/files/postinst
 rename meta/recipes-support/initramfs-fsck-hook/files/{postinst => initramfs-fsck-hook-ext4.triggers} (51%)
 rename meta/recipes-support/initramfs-fsck-hook/{initramfs-fsck-hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} (88%)

diff --git a/meta/classes/debianize.bbclass b/meta/classes/debianize.bbclass
index df7c5b50..25ae2023 100644
--- a/meta/classes/debianize.bbclass
+++ b/meta/classes/debianize.bbclass
@@ -138,4 +138,12 @@ deb_debianize() {
 			fi
 		done
 	done
+
+	# handle PN.service and PN.triggers files for use with debhelper
+	for f in service triggers
+	do
+		if [ -f ${WORKDIR}/${PN}.${f} ]; then
+			install -v -m 644 ${WORKDIR}/${PN}.${f} ${S}/debian/
+		fi
+	done
 }
diff --git a/meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.4.bb b/meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.5.bb
similarity index 78%
rename from meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.4.bb
rename to meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.5.bb
index 0996000c..1b5b066b 100644
--- a/meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.4.bb
+++ b/meta/recipes-support/expand-on-first-boot/expand-on-first-boot_1.5.bb
@@ -14,13 +14,9 @@ DEBIAN_DEPENDS = "systemd, sed, grep, coreutils, mount, e2fsprogs, fdisk (>=2.29
 
 SRC_URI = " \
     file://expand-on-first-boot.service \
-    file://expand-last-partition.sh \
-    file://postinst"
+    file://expand-last-partition.sh"
 
 do_install() {
-    install -d -m 755 ${D}/lib/systemd/system
-    install -m 644 ${WORKDIR}/expand-on-first-boot.service ${D}/lib/systemd/system/
-
     install -d -m 755 ${D}/usr/share/expand-on-first-boot
     install -m 755 ${WORKDIR}/expand-last-partition.sh ${D}/usr/share/expand-on-first-boot/
 }
diff --git a/meta/recipes-support/expand-on-first-boot/files/postinst b/meta/recipes-support/expand-on-first-boot/files/postinst
deleted file mode 100644
index a190b01a..00000000
--- a/meta/recipes-support/expand-on-first-boot/files/postinst
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-systemctl enable expand-on-first-boot.service
diff --git a/meta/recipes-support/initramfs-fsck-hook/files/postinst b/meta/recipes-support/initramfs-fsck-hook/files/initramfs-fsck-hook-ext4.triggers
similarity index 51%
rename from meta/recipes-support/initramfs-fsck-hook/files/postinst
rename to meta/recipes-support/initramfs-fsck-hook/files/initramfs-fsck-hook-ext4.triggers
index 07bc3d94..db45dfdc 100644
--- a/meta/recipes-support/initramfs-fsck-hook/files/postinst
+++ b/meta/recipes-support/initramfs-fsck-hook/files/initramfs-fsck-hook-ext4.triggers
@@ -1,6 +1,5 @@
-#!/bin/sh
 # This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
+# Copyright (C) Siemens AG, 2023
 #
 # SPDX-License-Identifier: MIT
-update-initramfs -u
+activate-await update-initramfs
diff --git a/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.1.bb b/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.2.bb
similarity index 88%
rename from meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.1.bb
rename to meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.2.bb
index 0be52a3b..9860dbc8 100644
--- a/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.1.bb
+++ b/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-ext4_0.2.bb
@@ -7,7 +7,7 @@
 DESCRIPTION = "Recipe to add fsck hook to the initramfs"
 
 inherit dpkg-raw
-SRC_URI = "file://postinst \
+SRC_URI = "file://initramfs-fsck-hook-ext4.triggers \
            file://initramfs.fsck.ext4.hook \
           "
 
-- 
2.39.2

Re: [PATCH] debianize: handle .triggers and .service files

[MOESSBAUER, Felix]

On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-users
wrote:
> Some packages need to update/rebuild the initramfs and this is
> particularly
> slow on Debian systems (which are not using more modern technologies
> such
> as dracut or mkosi). Instead of having each package call update-
> initramfs,
> use a trigger instead to have dpkg defer that call to the very end of
> the
> transaction. Many packages also failed to rebuild the initramfs when
> the
> package gets removed. Demonstrate use with initramfs-fsck-hook-ext4.

Hi Cedric,

thanks for bringing this up. Especially on non x86 builds the update-
initramfs can take really long due to the emulation.

I also played around with disabling the initrd update during image
building of images with a custom initramfs (where the one from the
rootfs is anyways not used). However the issue is, that the config
change needs to be made in /etc/initramfs-tools/update-initramfs.conf,
which is shipped by initramfs-tools (no conf.d style supported). By
that, we would need to add this package to the default set of packages
which I also dislike.

What I don't understand is how the initrd trigger should help. The
/usr/sbin/update-initramfs internally already detects the invocation
via dpkg and dispatches to the trigger, so manually adding this is nice
syntax wise, but does not make a difference in performance.

Ideally, we don't update the initrd at all during package install, but
just generate it at do_rootfs_postprocess.

Felix

> 
> Other packages ship systemd units and usually do this using
> do_install()
> and a postinst script. Such recipes also fail to handle the removal
> of
> such packages. By naming the unit ${PN}.service and putting it under
> debian/, debhelper will generate the necessary maintainer scripts
> (preinst, postinst, prerm, postrm).
> 
> Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
> ---
>  meta/classes/debianize.bbclass                            | 8
> ++++++++
>  ...d-on-first-boot_1.4.bb => expand-on-first-boot_1.5.bb} | 6 +-----
>  meta/recipes-support/expand-on-first-boot/files/postinst  | 3 ---
>  .../files/{postinst => initramfs-fsck-hook-ext4.triggers} | 5 ++---
>  ...k-hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} | 2 +-
>  5 files changed, 12 insertions(+), 12 deletions(-)
>  rename meta/recipes-support/expand-on-first-boot/{expand-on-first-
> boot_1.4.bb => expand-on-first-boot_1.5.bb} (78%)
>  delete mode 100644 meta/recipes-support/expand-on-first-
> boot/files/postinst
>  rename meta/recipes-support/initramfs-fsck-hook/files/{postinst =>
> initramfs-fsck-hook-ext4.triggers} (51%)
>  rename meta/recipes-support/initramfs-fsck-hook/{initramfs-fsck-
> hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} (88%)
> 
> diff --git a/meta/classes/debianize.bbclass
> b/meta/classes/debianize.bbclass
> index df7c5b50..25ae2023 100644
> --- a/meta/classes/debianize.bbclass
> +++ b/meta/classes/debianize.bbclass
> @@ -138,4 +138,12 @@ deb_debianize() {
>                         fi
>                 done
>         done
> +
> +       # handle PN.service and PN.triggers files for use with
> debhelper
> +       for f in service triggers
> +       do
> +               if [ -f ${WORKDIR}/${PN}.${f} ]; then
> +                       install -v -m 644 ${WORKDIR}/${PN}.${f}
> ${S}/debian/
> +               fi
> +       done
>  }
> diff --git a/meta/recipes-support/expand-on-first-boot/expand-on-
> first-boot_1.4.bb b/meta/recipes-support/expand-on-first-boot/expand-
> on-first-boot_1.5.bb
> similarity index 78%
> rename from meta/recipes-support/expand-on-first-boot/expand-on-
> first-boot_1.4.bb
> rename to meta/recipes-support/expand-on-first-boot/expand-on-first-
> boot_1.5.bb
> index 0996000c..1b5b066b 100644
> --- a/meta/recipes-support/expand-on-first-boot/expand-on-first-
> boot_1.4.bb
> +++ b/meta/recipes-support/expand-on-first-boot/expand-on-first-
> boot_1.5.bb
> @@ -14,13 +14,9 @@ DEBIAN_DEPENDS = "systemd, sed, grep, coreutils,
> mount, e2fsprogs, fdisk (>=2.29
>  
>  SRC_URI = " \
>      file://expand-on-first-boot.service \
> -    file://expand-last-partition.sh \
> -    file://postinst"
> +    file://expand-last-partition.sh"
>  
>  do_install() {
> -    install -d -m 755 ${D}/lib/systemd/system
> -    install -m 644 ${WORKDIR}/expand-on-first-boot.service
> ${D}/lib/systemd/system/
> -
>      install -d -m 755 ${D}/usr/share/expand-on-first-boot
>      install -m 755 ${WORKDIR}/expand-last-partition.sh
> ${D}/usr/share/expand-on-first-boot/
>  }
> diff --git a/meta/recipes-support/expand-on-first-boot/files/postinst
> b/meta/recipes-support/expand-on-first-boot/files/postinst
> deleted file mode 100644
> index a190b01a..00000000
> --- a/meta/recipes-support/expand-on-first-boot/files/postinst
> +++ /dev/null
> @@ -1,3 +0,0 @@
> -#!/bin/sh
> -
> -systemctl enable expand-on-first-boot.service
> diff --git a/meta/recipes-support/initramfs-fsck-hook/files/postinst
> b/meta/recipes-support/initramfs-fsck-hook/files/initramfs-fsck-hook-
> ext4.triggers
> similarity index 51%
> rename from meta/recipes-support/initramfs-fsck-hook/files/postinst
> rename to meta/recipes-support/initramfs-fsck-hook/files/initramfs-
> fsck-hook-ext4.triggers
> index 07bc3d94..db45dfdc 100644
> --- a/meta/recipes-support/initramfs-fsck-hook/files/postinst
> +++ b/meta/recipes-support/initramfs-fsck-hook/files/initramfs-fsck-
> hook-ext4.triggers
> @@ -1,6 +1,5 @@
> -#!/bin/sh
>  # This software is a part of ISAR.
> -# Copyright (C) Siemens AG, 2019
> +# Copyright (C) Siemens AG, 2023
>  #
>  # SPDX-License-Identifier: MIT
> -update-initramfs -u
> +activate-await update-initramfs
> diff --git a/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-
> hook-ext4_0.1.bb b/meta/recipes-support/initramfs-fsck-
> hook/initramfs-fsck-hook-ext4_0.2.bb
> similarity index 88%
> rename from meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-
> hook-ext4_0.1.bb
> rename to meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-
> hook-ext4_0.2.bb
> index 0be52a3b..9860dbc8 100644
> --- a/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-
> ext4_0.1.bb
> +++ b/meta/recipes-support/initramfs-fsck-hook/initramfs-fsck-hook-
> ext4_0.2.bb
> @@ -7,7 +7,7 @@
>  DESCRIPTION = "Recipe to add fsck hook to the initramfs"
>  
>  inherit dpkg-raw
> -SRC_URI = "file://postinst \
> +SRC_URI = "file://initramfs-fsck-hook-ext4.triggers \
>             file://initramfs.fsck.ext4.hook \
>            "
>  
> -- 
> 2.39.2
> 

Re: [PATCH] debianize: handle .triggers and .service files

[Jan Kiszka]

On 14.11.23 21:22, 'MOESSBAUER, Felix' via isar-users wrote:
> On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-users
> wrote:
>> Some packages need to update/rebuild the initramfs and this is
>> particularly
>> slow on Debian systems (which are not using more modern technologies
>> such
>> as dracut or mkosi). Instead of having each package call update-
>> initramfs,
>> use a trigger instead to have dpkg defer that call to the very end of
>> the
>> transaction. Many packages also failed to rebuild the initramfs when
>> the
>> package gets removed. Demonstrate use with initramfs-fsck-hook-ext4.
> 
> Hi Cedric,
> 
> thanks for bringing this up. Especially on non x86 builds the update-
> initramfs can take really long due to the emulation.
> 
> I also played around with disabling the initrd update during image
> building of images with a custom initramfs (where the one from the
> rootfs is anyways not used). However the issue is, that the config
> change needs to be made in /etc/initramfs-tools/update-initramfs.conf,
> which is shipped by initramfs-tools (no conf.d style supported). By
> that, we would need to add this package to the default set of packages
> which I also dislike.
> 
> What I don't understand is how the initrd trigger should help. The
> /usr/sbin/update-initramfs internally already detects the invocation
> via dpkg and dispatches to the trigger, so manually adding this is nice
> syntax wise, but does not make a difference in performance.
> 
> Ideally, we don't update the initrd at all during package install, but
> just generate it at do_rootfs_postprocess.
> 

There is actually an issue already in Debian itself which I started to
examine during Debconf but was distracted: If you install a minimal
rootfs with kernel, that will already run the initramfs generation
twice, although it should do it once only. That said, anything on top
may be related to missing proper usage of triggers.

Jan

-- 
Siemens AG, Technology
Linux Expert Center

Re: [PATCH] debianize: handle .triggers and .service files

[MOESSBAUER, Felix]

On Wed, 2023-11-15 at 09:36 +0100, Jan Kiszka wrote:
> On 14.11.23 21:22, 'MOESSBAUER, Felix' via isar-users wrote:
> > On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-
> > users
> > wrote:
> > > Some packages need to update/rebuild the initramfs and this is
> > > particularly
> > > slow on Debian systems (which are not using more modern
> > > technologies
> > > such
> > > as dracut or mkosi). Instead of having each package call update-
> > > initramfs,
> > > use a trigger instead to have dpkg defer that call to the very
> > > end of
> > > the
> > > transaction. Many packages also failed to rebuild the initramfs
> > > when
> > > the
> > > package gets removed. Demonstrate use with initramfs-fsck-hook-
> > > ext4.
> > 
> > Hi Cedric,
> > 
> > thanks for bringing this up. Especially on non x86 builds the
> > update-
> > initramfs can take really long due to the emulation.
> > 
> > I also played around with disabling the initrd update during image
> > building of images with a custom initramfs (where the one from the
> > rootfs is anyways not used). However the issue is, that the config
> > change needs to be made in /etc/initramfs-tools/update-
> > initramfs.conf,
> > which is shipped by initramfs-tools (no conf.d style supported). By
> > that, we would need to add this package to the default set of
> > packages
> > which I also dislike.
> > 
> > What I don't understand is how the initrd trigger should help. The
> > /usr/sbin/update-initramfs internally already detects the
> > invocation
> > via dpkg and dispatches to the trigger, so manually adding this is
> > nice
> > syntax wise, but does not make a difference in performance.
> > 
> > Ideally, we don't update the initrd at all during package install,
> > but
> > just generate it at do_rootfs_postprocess.
> > 
> 
> There is actually an issue already in Debian itself which I started
> to
> examine during Debconf but was distracted: If you install a minimal
> rootfs with kernel, that will already run the initramfs generation
> twice, although it should do it once only. That said, anything on top
> may be related to missing proper usage of triggers.

Ok... can you provide a link to the issue?

Regarding the triggers: The triggers itself work properly, but they are
called once per apt install/remove invocation, and as we have multiple
apt calls in ISAR (within a single build), also the triggers are called
multiple times. We really need to fully disable the initramfs
generation and call it manually at the end of the rootfs install.

Felix

> 
> Jan
> 

Re: [PATCH] debianize: handle .triggers and .service files

[cedric.hombourger]

On 11/15/23 09:43, MOESSBAUER, Felix (T CED INW-CN) wrote:
> On Wed, 2023-11-15 at 09:36 +0100, Jan Kiszka wrote:
>> On 14.11.23 21:22, 'MOESSBAUER, Felix' via isar-users wrote:
>>> On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-
>>> users
>>> wrote:
>>>> Some packages need to update/rebuild the initramfs and this is
>>>> particularly
>>>> slow on Debian systems (which are not using more modern
>>>> technologies
>>>> such
>>>> as dracut or mkosi). Instead of having each package call update-
>>>> initramfs,
>>>> use a trigger instead to have dpkg defer that call to the very
>>>> end of
>>>> the
>>>> transaction. Many packages also failed to rebuild the initramfs
>>>> when
>>>> the
>>>> package gets removed. Demonstrate use with initramfs-fsck-hook-
>>>> ext4.
>>> Hi Cedric,
Hi Felix
>>> thanks for bringing this up. Especially on non x86 builds the
>>> update-
>>> initramfs can take really long due to the emulation.
>>>
>>> I also played around with disabling the initrd update during image
>>> building of images with a custom initramfs (where the one from the
>>> rootfs is anyways not used). However the issue is, that the config
>>> change needs to be made in /etc/initramfs-tools/update-
>>> initramfs.conf,
>>> which is shipped by initramfs-tools (no conf.d style supported). By
>>> that, we would need to add this package to the default set of
>>> packages
>>> which I also dislike.
I am wondering if we should provide a pseudo replacement for 
initramfs-tools in images where we will end-up using an initramfs from 
an 'inherit initramfs' recipe
>>> What I don't understand is how the initrd trigger should help. The
>>> /usr/sbin/update-initramfs internally already detects the
>>> invocation
>>> via dpkg and dispatches to the trigger, so manually adding this is
>>> nice
>>> syntax wise, but does not make a difference in performance.
>>>
>>> Ideally, we don't update the initrd at all during package install,
>>> but
>>> just generate it at do_rootfs_postprocess.
>>>
>> There is actually an issue already in Debian itself which I started
>> to
>> examine during Debconf but was distracted: If you install a minimal
>> rootfs with kernel, that will already run the initramfs generation
>> twice, although it should do it once only. That said, anything on top
>> may be related to missing proper usage of triggers.
> Ok... can you provide a link to the issue?
>
> Regarding the triggers: The triggers itself work properly, but they are
> called once per apt install/remove invocation, and as we have multiple
> apt calls in ISAR (within a single build), also the triggers are called
> multiple times. We really need to fully disable the initramfs
> generation and call it manually at the end of the rootfs install.

Agreed. There are however (and IMO) two sides of the story:

(1) encouraging people to use debhelper and best practices when they 
create Debian packages (with or without dpkg-raw). I have downstream 
users relying on the package feeds we generate from Isar and who either 
pull packages from an Isar build or from a running system (for those, 
packages we provide should meet Debian standards)

(2) improving the Isar image generation pipeline

Our Isar-based DISTRO switched to triggers quite a while ago and it was 
definitely a technical debt from our part not to submit that dpkg-raw 
enhancement earlier. I seem to recall that when we had started with 
buster, update-initramfs did not check if it was running in maintainer 
context and would not defer the update of the initramfs to the very end 
of the apt transaction. As such it is very likely that this proposed 
change does not improve build-times but I hope we would agree that it 
would result in initrd package recipes ship proper maintainer scripts 
(most that I have seen were only taking care of the postinst step).

> Felix
>
>> Jan
>>

Re: [PATCH] debianize: handle .triggers and .service files

[MOESSBAUER, Felix]

On Wed, 2023-11-15 at 08:57 +0000, Hombourger, Cedric (DI SW CAS ES LI)
wrote:
> On 11/15/23 09:43, MOESSBAUER, Felix (T CED INW-CN) wrote:
> > On Wed, 2023-11-15 at 09:36 +0100, Jan Kiszka wrote:
> > > On 14.11.23 21:22, 'MOESSBAUER, Felix' via isar-users wrote:
> > > > On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via
> > > > isar-
> > > > users
> > > > wrote:
> > > > > Some packages need to update/rebuild the initramfs and this
> > > > > is
> > > > > particularly
> > > > > slow on Debian systems (which are not using more modern
> > > > > technologies
> > > > > such
> > > > > as dracut or mkosi). Instead of having each package call
> > > > > update-
> > > > > initramfs,
> > > > > use a trigger instead to have dpkg defer that call to the
> > > > > very
> > > > > end of
> > > > > the
> > > > > transaction. Many packages also failed to rebuild the
> > > > > initramfs
> > > > > when
> > > > > the
> > > > > package gets removed. Demonstrate use with initramfs-fsck-
> > > > > hook-
> > > > > ext4.
> > > > Hi Cedric,
> Hi Felix
> > > > thanks for bringing this up. Especially on non x86 builds the
> > > > update-
> > > > initramfs can take really long due to the emulation.
> > > > 
> > > > I also played around with disabling the initrd update during
> > > > image
> > > > building of images with a custom initramfs (where the one from
> > > > the
> > > > rootfs is anyways not used). However the issue is, that the
> > > > config
> > > > change needs to be made in /etc/initramfs-tools/update-
> > > > initramfs.conf,
> > > > which is shipped by initramfs-tools (no conf.d style
> > > > supported). By
> > > > that, we would need to add this package to the default set of
> > > > packages
> > > > which I also dislike.
> I am wondering if we should provide a pseudo replacement for
> initramfs-tools in images where we will end-up using an initramfs
> from
> an 'inherit initramfs' recipe
> > > > What I don't understand is how the initrd trigger should help.
> > > > The
> > > > /usr/sbin/update-initramfs internally already detects the
> > > > invocation
> > > > via dpkg and dispatches to the trigger, so manually adding this
> > > > is
> > > > nice
> > > > syntax wise, but does not make a difference in performance.
> > > > 
> > > > Ideally, we don't update the initrd at all during package
> > > > install,
> > > > but
> > > > just generate it at do_rootfs_postprocess.
> > > > 
> > > There is actually an issue already in Debian itself which I
> > > started
> > > to
> > > examine during Debconf but was distracted: If you install a
> > > minimal
> > > rootfs with kernel, that will already run the initramfs
> > > generation
> > > twice, although it should do it once only. That said, anything on
> > > top
> > > may be related to missing proper usage of triggers.
> > Ok... can you provide a link to the issue?
> > 
> > Regarding the triggers: The triggers itself work properly, but they
> > are
> > called once per apt install/remove invocation, and as we have
> > multiple
> > apt calls in ISAR (within a single build), also the triggers are
> > called
> > multiple times. We really need to fully disable the initramfs
> > generation and call it manually at the end of the rootfs install.
> 
> Agreed. There are however (and IMO) two sides of the story:
> 
> (1) encouraging people to use debhelper and best practices when they
> create Debian packages (with or without dpkg-raw). I have downstream
> users relying on the package feeds we generate from Isar and who
> either
> pull packages from an Isar build or from a running system (for those,
> packages we provide should meet Debian standards)
> 
> (2) improving the Isar image generation pipeline
> 
> Our Isar-based DISTRO switched to triggers quite a while ago and it
> was
> definitely a technical debt from our part not to submit that dpkg-raw
> enhancement earlier. I seem to recall that when we had started with
> buster, update-initramfs did not check if it was running in
> maintainer
> context and would not defer the update of the initramfs to the very
> end
> of the apt transaction. As such it is very likely that this proposed
> change does not improve build-times but I hope we would agree that it
> would result in initrd package recipes ship proper maintainer scripts
> (most that I have seen were only taking care of the postinst step).

Yes. I fully agree that this patch makes sense and we should include
it. This discussion should also not delay the integration.

Reviewed-by: Felix Moessbauer <felix.moessbauer@siemens.com>

Apart from that, I would be happy if we could solve the performance
issue on a broader scope. But this will need time to work on the
required changes and likely also more discussions with upstream debian.

Felix

> 
> > Felix
> > 
> > > Jan
> > > 
> 

Re: [PATCH] debianize: handle .triggers and .service files

[Jan Kiszka]

On 15.11.23 09:43, MOESSBAUER, Felix (T CED INW-CN) wrote:
> On Wed, 2023-11-15 at 09:36 +0100, Jan Kiszka wrote:
>> On 14.11.23 21:22, 'MOESSBAUER, Felix' via isar-users wrote:
>>> On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-
>>> users
>>> wrote:
>>>> Some packages need to update/rebuild the initramfs and this is
>>>> particularly
>>>> slow on Debian systems (which are not using more modern
>>>> technologies
>>>> such
>>>> as dracut or mkosi). Instead of having each package call update-
>>>> initramfs,
>>>> use a trigger instead to have dpkg defer that call to the very
>>>> end of
>>>> the
>>>> transaction. Many packages also failed to rebuild the initramfs
>>>> when
>>>> the
>>>> package gets removed. Demonstrate use with initramfs-fsck-hook-
>>>> ext4.
>>>
>>> Hi Cedric,
>>>
>>> thanks for bringing this up. Especially on non x86 builds the
>>> update-
>>> initramfs can take really long due to the emulation.
>>>
>>> I also played around with disabling the initrd update during image
>>> building of images with a custom initramfs (where the one from the
>>> rootfs is anyways not used). However the issue is, that the config
>>> change needs to be made in /etc/initramfs-tools/update-
>>> initramfs.conf,
>>> which is shipped by initramfs-tools (no conf.d style supported). By
>>> that, we would need to add this package to the default set of
>>> packages
>>> which I also dislike.
>>>
>>> What I don't understand is how the initrd trigger should help. The
>>> /usr/sbin/update-initramfs internally already detects the
>>> invocation
>>> via dpkg and dispatches to the trigger, so manually adding this is
>>> nice
>>> syntax wise, but does not make a difference in performance.
>>>
>>> Ideally, we don't update the initrd at all during package install,
>>> but
>>> just generate it at do_rootfs_postprocess.
>>>
>>
>> There is actually an issue already in Debian itself which I started
>> to
>> examine during Debconf but was distracted: If you install a minimal
>> rootfs with kernel, that will already run the initramfs generation
>> twice, although it should do it once only. That said, anything on top
>> may be related to missing proper usage of triggers.
> 
> Ok... can you provide a link to the issue?

Nope, not yet reported. But you can just take a debian:bookworm-slim 
container and install linux-image-amd64:

...
Setting up initramfs-tools-core (0.142) ...
Setting up initramfs-tools (0.142) ...
update-initramfs: deferring update (trigger activated)
Setting up linux-image-6.1.0-13-amd64 (6.1.55-1) ...
I: /vmlinuz.old is now a symlink to boot/vmlinuz-6.1.0-13-amd64
I: /initrd.img.old is now a symlink to boot/initrd.img-6.1.0-13-amd64
I: /vmlinuz is now a symlink to boot/vmlinuz-6.1.0-13-amd64
I: /initrd.img is now a symlink to boot/initrd.img-6.1.0-13-amd64
/etc/kernel/postinst.d/initramfs-tools:
update-initramfs: Generating /boot/initrd.img-6.1.0-13-amd64
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8156b-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8156a-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153c-1.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153b-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-4.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-3.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-2.fw for module r8152
I: The initramfs will attempt to resume from /dev/dm-2
I: Set the RESUME variable to override this.
Setting up linux-image-amd64 (6.1.55-1) ...
Processing triggers for libc-bin (2.36-8) ...
Processing triggers for initramfs-tools (0.142) ...
update-initramfs: Generating /boot/initrd.img-6.1.0-13-amd64
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8156b-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8156a-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153c-1.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153b-2.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-4.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-3.fw for module r8152
W: Possible missing firmware /lib/firmware/rtl_nic/rtl8153a-2.fw for module r8152
I: The initramfs will attempt to resume from /dev/dm-2
I: Set the RESUME variable to override this.

The kernel package enforces the initramfs generation, and then it is 
done again while handling the pending triggers.

Jan

-- 
Siemens AG, Technology
Linux Expert Center

Re: [PATCH] debianize: handle .triggers and .service files

[Uladzimir Bely]

On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-users
wrote:
> Some packages need to update/rebuild the initramfs and this is
> particularly
> slow on Debian systems (which are not using more modern technologies
> such
> as dracut or mkosi). Instead of having each package call update-
> initramfs,
> use a trigger instead to have dpkg defer that call to the very end of
> the
> transaction. Many packages also failed to rebuild the initramfs when
> the
> package gets removed. Demonstrate use with initramfs-fsck-hook-ext4.
> 
> Other packages ship systemd units and usually do this using
> do_install()
> and a postinst script. Such recipes also fail to handle the removal
> of
> such packages. By naming the unit ${PN}.service and putting it under
> debian/, debhelper will generate the necessary maintainer scripts
> (preinst, postinst, prerm, postrm).
> 
> Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
> ---
>  meta/classes/debianize.bbclass                            | 8
> ++++++++
>  ...d-on-first-boot_1.4.bb => expand-on-first-boot_1.5.bb} | 6 +-----
>  meta/recipes-support/expand-on-first-boot/files/postinst  | 3 ---
>  .../files/{postinst => initramfs-fsck-hook-ext4.triggers} | 5 ++---
>  ...k-hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} | 2 +-
>  5 files changed, 12 insertions(+), 12 deletions(-)

The discusstion looks a bit unfinished. But the patch itself passes CI
and we are ready to merge it if there are no objections.

Re: [PATCH] debianize: handle .triggers and .service files

[Uladzimir Bely]

On Tue, 2023-11-14 at 20:54 +0100, 'Cedric Hombourger' via isar-users
wrote:
> Some packages need to update/rebuild the initramfs and this is
> particularly
> slow on Debian systems (which are not using more modern technologies
> such
> as dracut or mkosi). Instead of having each package call update-
> initramfs,
> use a trigger instead to have dpkg defer that call to the very end of
> the
> transaction. Many packages also failed to rebuild the initramfs when
> the
> package gets removed. Demonstrate use with initramfs-fsck-hook-ext4.
> 
> Other packages ship systemd units and usually do this using
> do_install()
> and a postinst script. Such recipes also fail to handle the removal
> of
> such packages. By naming the unit ${PN}.service and putting it under
> debian/, debhelper will generate the necessary maintainer scripts
> (preinst, postinst, prerm, postrm).
> 
> Signed-off-by: Cedric Hombourger <cedric.hombourger@siemens.com>
> ---
>  meta/classes/debianize.bbclass                            | 8
> ++++++++
>  ...d-on-first-boot_1.4.bb => expand-on-first-boot_1.5.bb} | 6 +-----
>  meta/recipes-support/expand-on-first-boot/files/postinst  | 3 ---
>  .../files/{postinst => initramfs-fsck-hook-ext4.triggers} | 5 ++---
>  ...k-hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} | 2 +-
>  5 files changed, 12 insertions(+), 12 deletions(-)
>  rename meta/recipes-support/expand-on-first-boot/{expand-on-first-
> boot_1.4.bb => expand-on-first-boot_1.5.bb} (78%)
>  delete mode 100644 meta/recipes-support/expand-on-first-
> boot/files/postinst
>  rename meta/recipes-support/initramfs-fsck-hook/files/{postinst =>
> initramfs-fsck-hook-ext4.triggers} (51%)
>  rename meta/recipes-support/initramfs-fsck-hook/{initramfs-fsck-
> hook-ext4_0.1.bb => initramfs-fsck-hook-ext4_0.2.bb} (88%)
> 
Applied to next, thanks.