RE: [PATCH v3 4/7] imagetypes: restructure files

["Moessbauer, Felix" <felix.moessbauer@siemens.com>]

> +
> +# image type: ubi
> +IMAGER_INSTALL_ubi += "mtd-utils"
> +IMAGE_CMD_REQUIRED_ARGS_ubi = "UBINIZE_ARGS"
> +UBINIZE_CFG ??= "ubinize.cfg"
> +
> +IMAGE_CMD_ubi() {
> +    if [ ! -e "${WORKDIR}/${UBINIZE_CFG}" ]; then
> +        die "UBINIZE_CFG does not contain ubinize config file."
> +    fi
> +
> +    ${SUDO_CHROOT} /usr/sbin/ubinize ${UBINIZE_ARGS} \
> +                -o '${IMAGE_FILE_CHROOT}' '${PP_WORK}/${UBINIZE_CFG}'
> +}
> +IMAGE_CMD_ubi[depends] = "${PN}:do_transform_template"
> +
> +# image conversions
> +IMAGE_CONVERSIONS = "gz xz"
> +
> +CONVERSION_CMD_gz = "${SUDO_CHROOT} sh -c 'gzip -f -9 -n -c --rsyncable
> ${IMAGE_FILE_CHROOT} > ${IMAGE_FILE_CHROOT}.gz'"
> +CONVERSION_DEPS_gz = "gzip"
> +
> +XZ_OPTIONS ?= ""
> +CONVERSION_CMD_xz = "${SUDO_CHROOT} sh -c 'cat
> ${IMAGE_FILE_CHROOT} | xz ${XZ_OPTIONS} > ${IMAGE_FILE_CHROOT}.xz'"

This breaks parallel compression as the pipe feeds in data in small chunks.
Writing this as following gives a super linear speedup and even better compression ratios (for large images).
But without limiting memory, we might easily get OOMs.
Please also note, that xz using default settings does not compress in parallel (at least the version on Debian bullseye).

XZ_OPTIONS ?= "-T0"
CONVERSION_CMD_xz = "${SUDO_CHROOT} sh -c 'xz -k -c ${XZ_OPTIONS} ${IMAGE_FILE_CHROOT} > ${IMAGE_FILE_CHROOT}.xz'"

Felix

> +CONVERSION_DEPS_xz = "xz-utils"
> diff --git a/meta/classes/container-img.bbclass
> b/meta/classes/imagetypes_container.bbclass
> similarity index 100%
> rename from meta/classes/container-img.bbclass
> rename to meta/classes/imagetypes_container.bbclass
> diff --git a/meta/classes/vm-img.bbclass
> b/meta/classes/imagetypes_vm.bbclass
> similarity index 100%
> rename from meta/classes/vm-img.bbclass
> rename to meta/classes/imagetypes_vm.bbclass
> diff --git a/meta/classes/wic-img.bbclass
> b/meta/classes/imagetypes_wic.bbclass
> similarity index 100%
> rename from meta/classes/wic-img.bbclass rename to
> meta/classes/imagetypes_wic.bbclass
> diff --git a/meta/classes/targz-img.bbclass b/meta/classes/targz-img.bbclass
> deleted file mode 100644 index 74d34e29..00000000
> --- a/meta/classes/targz-img.bbclass
> +++ /dev/null
> @@ -1,8 +0,0 @@
> -# This software is a part of ISAR.
> -# Copyright (c) Siemens AG, 2018
> -#
> -# SPDX-License-Identifier: MIT
> -
> -IMAGE_CMD_tar() {
> -    sudo tar -cvzf ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.tar.gz --one-
> file-system -C ${IMAGE_ROOTFS} .
> -}
> diff --git a/meta/classes/ubi-img.bbclass b/meta/classes/ubi-img.bbclass
> deleted file mode 100644 index 92acb6f8..00000000
> --- a/meta/classes/ubi-img.bbclass
> +++ /dev/null
> @@ -1,20 +0,0 @@
> -# This software is a part of ISAR.
> -# Copyright (C) Siemens AG, 2019
> -#
> -# SPDX-License-Identifier: MIT
> -
> -UBINIZE_CFG ??= "ubinize.cfg"
> -
> -IMAGER_INSTALL_ubi += "mtd-utils"
> -
> -# Generate ubi filesystem image
> -IMAGE_CMD_ubi() {
> -    if [ ! -e "${WORKDIR}/${UBINIZE_CFG}" ]; then
> -        die "UBINIZE_CFG does not contain ubinize config file."
> -    fi
> -
> -    ${SUDO_CHROOT} /usr/sbin/ubinize ${UBINIZE_ARGS} \
> -                -o '${IMAGE_FILE_CHROOT}' '${PP_WORK}/${UBINIZE_CFG}'
> -}
> -IMAGE_CMD_ubi[depends] = "${PN}:do_transform_template"
> -IMAGE_CMD_REQUIRED_ARGS_ubi = "UBINIZE_ARGS"
> diff --git a/meta/classes/ubifs-img.bbclass b/meta/classes/ubifs-img.bbclass
> deleted file mode 100644 index 5f1be89e..00000000
> --- a/meta/classes/ubifs-img.bbclass
> +++ /dev/null
> @@ -1,19 +0,0 @@
> -# This software is a part of ISAR.
> -# Copyright (C) Siemens AG, 2019
> -#
> -# SPDX-License-Identifier: MIT
> -
> -IMAGER_INSTALL_ubifs += "mtd-utils"
> -
> -# glibc bug 23960
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsourcew
> are.org%2Fbugzilla%2Fshow_bug.cgi%3Fid%3D23960&data=04%7C01%7Cf
> elix.moessbauer%40siemens.com%7Ccae347b95ae94fabeac708da1ead1846%7
> C38ae3bcd95794fd4addab42e1495d55a%7C1%7C0%7C637856026794071339%
> 7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTi
> I6Ik1haWwiLCJXVCI6Mn0%3D%7C2000&sdata=EpBCpH7vBfCKj%2B5MTYc
> ORJKdupyPLu9HK%2F1uT93fHTc%3D&reserved=0
> -# should not use QEMU on armhf target with mkfs.ubifs < v2.1.3 -
> THIS_ISAR_CROSS_COMPILE := "${ISAR_CROSS_COMPILE}"
> -ISAR_CROSS_COMPILE_armhf = "${@bb.utils.contains('IMAGE_BASETYPES',
> 'ubifs', '1', '${THIS_ISAR_CROSS_COMPILE}', d)}"
> -
> -# Generate ubifs filesystem image
> -IMAGE_CMD_ubifs() {
> -    # Create ubifs image using buildchroot tools
> -    ${SUDO_CHROOT} /usr/sbin/mkfs.ubifs ${MKUBIFS_ARGS} \
> -                -r '${PP_ROOTFS}' '${IMAGE_FILE_CHROOT}'
> -}
> -IMAGE_CMD_REQUIRED_ARGS_ubifs = "MKUBIFS_ARGS"
> --
> 2.30.2
> 
> --
> You received this message because you are subscribed to the Google Groups
> "isar-users" group.
> To unsubscribe from this group and stop receiving emails from it, send an email
> to isar-users+unsubscribe@googlegroups.com.
> To view this discussion on the web visit
> https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgroups.g
> oogle.com%2Fd%2Fmsgid%2Fisar-users%2F20220415065617.1910954-5-
> adriaan.schmidt%2540siemens.com&amp;data=04%7C01%7Cfelix.moessbauer
> %40siemens.com%7Ccae347b95ae94fabeac708da1ead1846%7C38ae3bcd9579
> 4fd4addab42e1495d55a%7C1%7C0%7C637856026794071339%7CUnknown%7C
> TWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> VCI6Mn0%3D%7C2000&amp;sdata=1MTK5T8Hp5RbSFPMtoEFDB2jHNmUgm7jR
> SWj3JdeUlE%3D&amp;reserved=0.