[PATCH] meta/classes: generate bill of material from image

[PATCH] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

To use it add following line to the local.conf:
ISAR_DO_PACKAGE_LIST ?= "1"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
 meta-isar/conf/local.conf.sample                 |  4 ++++
 .../classes/image-package-list-extension.bbclass | 16 ++++++++++++++++
 meta/classes/image.bbclass                       |  1 +
 3 files changed, 21 insertions(+)
 create mode 100644 meta/classes/image-package-list-extension.bbclass

diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 5b3a0a1..d188051 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsc
 # NOTE: this works on build host >= stretch for armhf, arm64 and amd64 targets for now.
 ISAR_CROSS_COMPILE ?= "0"
 
+#
+# Generate package list 
+ISAR_DO_PACKAGE_LIST ?= "1"
+
 #
 # Uncomment this to enable use of cached base repository
 #ISAR_USE_CACHED_BASE_REPO ?= "1"
diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
new file mode 100644
index 0000000..558922e
--- /dev/null
+++ b/meta/classes/image-package-list-extension.bbclass
@@ -0,0 +1,16 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+
+ISAR_DO_PACKAGE_LIST ??= "0"
+image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}"
+image_package_list() {
+    if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}" = 'True' ]; then
+      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
+          -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
+    fi
+}
+ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list"
+
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index ec6bd39..85bab64 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -68,6 +68,7 @@ inherit image-tools-extension
 inherit image-postproc-extension
 inherit image-locales-extension
 inherit image-account-extension
+inherit image-package-list-extension
 
 # Extra space for rootfs in MB
 ROOTFS_EXTRA ?= "64"
-- 
2.20.1

Re: [PATCH] meta/classes: generate bill of material from image

[vijai kumar]

[-- Attachment #1.1: Type: text/plain, Size: 3234 bytes --]

Ok. I have been looking for something like this for some time now :)


On Monday, August 5, 2019 at 7:38:31 PM UTC+5:30, Q. Gylstorff wrote:

> From: Quirin Gylstorff <quirin....@siemens.com <javascript:>> 
>
> To create products it is necessary to have a list 
> of used packages for clearance and to security monitoring. 
> To get a simple list of packages use dpkg-query and generate 
> a list with the following pattern: 
>
> source name| source version | binary package name | binary version 
>
> To use it add following line to the local.conf: 
> ISAR_DO_PACKAGE_LIST ?= "1" 
>
> Signed-off-by: Quirin Gylstorff <quirin....@siemens.com <javascript:>> 
> --- 
>  meta-isar/conf/local.conf.sample                 |  4 ++++ 
>  .../classes/image-package-list-extension.bbclass | 16 ++++++++++++++++ 
>  meta/classes/image.bbclass                       |  1 + 
>  3 files changed, 21 insertions(+) 
>  create mode 100644 meta/classes/image-package-list-extension.bbclass 
>
> diff --git a/meta-isar/conf/local.conf.sample 
> b/meta-isar/conf/local.conf.sample 
> index 5b3a0a1..d188051 100644 
> --- a/meta-isar/conf/local.conf.sample 
> +++ b/meta-isar/conf/local.conf.sample 
> @@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw 
> example-module-${KERNEL_NAME} enable-fsc 
>  # NOTE: this works on build host >= stretch for armhf, arm64 and amd64 
> targets for now. 
>  ISAR_CROSS_COMPILE ?= "0" 
>   
> +# 
> +# Generate package list 
> +ISAR_DO_PACKAGE_LIST ?= "1" 
> + 
>  # 
>  # Uncomment this to enable use of cached base repository 
>  #ISAR_USE_CACHED_BASE_REPO ?= "1" 
> diff --git a/meta/classes/image-package-list-extension.bbclass 
> b/meta/classes/image-package-list-extension.bbclass 
> new file mode 100644 
> index 0000000..558922e 
> --- /dev/null 
> +++ b/meta/classes/image-package-list-extension.bbclass 
> @@ -0,0 +1,16 @@ 
> +# This software is a part of ISAR. 
> +# Copyright (C) Siemens AG, 2019 
> +# 
> +# SPDX-License-Identifier: MIT 
> + 
> +ISAR_DO_PACKAGE_LIST ??= "0" 
> +image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}" 
> +image_package_list() { 
> +    if [ 
> "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}" = 'True' 
> ]; then 
> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \ 
> +          -f 
> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \ 
> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst 
>

May be we could format and sort this a bit.

dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
              -f 
'${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W | 
column -t | sort

You could add column headers as well.

+    fi 
> +} 
> +ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list" 
> + 
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass 
> index ec6bd39..85bab64 100644 
> --- a/meta/classes/image.bbclass 
> +++ b/meta/classes/image.bbclass 
> @@ -68,6 +68,7 @@ inherit image-tools-extension 
>  inherit image-postproc-extension 
>  inherit image-locales-extension 
>  inherit image-account-extension 
> +inherit image-package-list-extension 
>   
>  # Extra space for rootfs in MB 
>  ROOTFS_EXTRA ?= "64" 
> -- 
> 2.20.1 
>

Thanks,
Vijai Kumar K

[-- Attachment #1.2: Type: text/html, Size: 4509 bytes --]

Re: [PATCH] meta/classes: generate bill of material from image

[Henning Schild]

Hi Quirin,

do you have a real use-case for that? In fact we do have similar things
internally already but did not propose it yet. Reason is you still need
to find all the magic switches to collect the source and binary package
versions, and you most likely want a machine readable format.

In fact we should just fix that broken offline cache thing and enable
src-packages for it. That would generate a list that will hopefully be
complete and in a well defined format.

Without the real use-case and the post-processing this is just another
obscure Isar feature and i am against merging it.

Henning

Am Mon, 5 Aug 2019 16:07:42 +0200
schrieb "[ext] Q. Gylstorff" <Quirin.Gylstorff@siemens.com>:

> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> To use it add following line to the local.conf:
> ISAR_DO_PACKAGE_LIST ?= "1"
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
>  meta-isar/conf/local.conf.sample                 |  4 ++++
>  .../classes/image-package-list-extension.bbclass | 16
> ++++++++++++++++ meta/classes/image.bbclass                       |
> 1 + 3 files changed, 21 insertions(+)
>  create mode 100644 meta/classes/image-package-list-extension.bbclass
> 
> diff --git a/meta-isar/conf/local.conf.sample
> b/meta-isar/conf/local.conf.sample index 5b3a0a1..d188051 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw
> example-module-${KERNEL_NAME} enable-fsc # NOTE: this works on build
> host >= stretch for armhf, arm64 and amd64 targets for now.
> ISAR_CROSS_COMPILE ?= "0" 
> +#
> +# Generate package list 
> +ISAR_DO_PACKAGE_LIST ?= "1"
> +
>  #
>  # Uncomment this to enable use of cached base repository
>  #ISAR_USE_CACHED_BASE_REPO ?= "1"
> diff --git a/meta/classes/image-package-list-extension.bbclass
> b/meta/classes/image-package-list-extension.bbclass new file mode
> 100644 index 0000000..558922e
> --- /dev/null
> +++ b/meta/classes/image-package-list-extension.bbclass
> @@ -0,0 +1,16 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +
> +ISAR_DO_PACKAGE_LIST ??= "0"
> +image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}"
> +image_package_list() {
> +    if
> [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}" =
> 'True' ]; then
> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
> +          -f
> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n'
> -W > \
> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
> +    fi
> +}
> +ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list"
> +
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..85bab64 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -68,6 +68,7 @@ inherit image-tools-extension
>  inherit image-postproc-extension
>  inherit image-locales-extension
>  inherit image-account-extension
> +inherit image-package-list-extension
>  
>  # Extra space for rootfs in MB
>  ROOTFS_EXTRA ?= "64"

Re: [PATCH] meta/classes: generate bill of material from image

[Jan Kiszka]

On 05.08.19 16:43, [ext] Henning Schild wrote:
> Hi Quirin,
> 
> do you have a real use-case for that? In fact we do have similar things
> internally already but did not propose it yet. Reason is you still need
> to find all the magic switches to collect the source and binary package
> versions, and you most likely want a machine readable format.

The use cases are the same what we have that internal stuff for.

> 
> In fact we should just fix that broken offline cache thing and enable
> src-packages for it. That would generate a list that will hopefully be
> complete and in a well defined format.

Good points: The format should be easily parseable.

> 
> Without the real use-case and the post-processing this is just another
> obscure Isar feature and i am against merging it.

How would you do it otherwise? Tell the beginners to remember which command to
run with which switches on which subdirs?

Quirin and I were also discussing the first extension: We need to collect
information about additional packages that are not part of the rootfs but the
image, primarily bootloaders. Providing a framework - however that may look like
- for recipe authors to report them would be valuable and can be achieved by any
local solution that cleanly.

Jan

> 
> Henning
> 
> Am Mon, 5 Aug 2019 16:07:42 +0200
> schrieb "[ext] Q. Gylstorff" <Quirin.Gylstorff@siemens.com>:
> 
>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>
>> To create products it is necessary to have a list
>> of used packages for clearance and to security monitoring.
>> To get a simple list of packages use dpkg-query and generate
>> a list with the following pattern:
>>
>> source name| source version | binary package name | binary version
>>
>> To use it add following line to the local.conf:
>> ISAR_DO_PACKAGE_LIST ?= "1"
>>
>> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>> ---
>>  meta-isar/conf/local.conf.sample                 |  4 ++++
>>  .../classes/image-package-list-extension.bbclass | 16
>> ++++++++++++++++ meta/classes/image.bbclass                       |
>> 1 + 3 files changed, 21 insertions(+)
>>  create mode 100644 meta/classes/image-package-list-extension.bbclass
>>
>> diff --git a/meta-isar/conf/local.conf.sample
>> b/meta-isar/conf/local.conf.sample index 5b3a0a1..d188051 100644
>> --- a/meta-isar/conf/local.conf.sample
>> +++ b/meta-isar/conf/local.conf.sample
>> @@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw
>> example-module-${KERNEL_NAME} enable-fsc # NOTE: this works on build
>> host >= stretch for armhf, arm64 and amd64 targets for now.
>> ISAR_CROSS_COMPILE ?= "0" 
>> +#
>> +# Generate package list 
>> +ISAR_DO_PACKAGE_LIST ?= "1"
>> +
>>  #
>>  # Uncomment this to enable use of cached base repository
>>  #ISAR_USE_CACHED_BASE_REPO ?= "1"
>> diff --git a/meta/classes/image-package-list-extension.bbclass
>> b/meta/classes/image-package-list-extension.bbclass new file mode
>> 100644 index 0000000..558922e
>> --- /dev/null
>> +++ b/meta/classes/image-package-list-extension.bbclass
>> @@ -0,0 +1,16 @@
>> +# This software is a part of ISAR.
>> +# Copyright (C) Siemens AG, 2019
>> +#
>> +# SPDX-License-Identifier: MIT
>> +
>> +ISAR_DO_PACKAGE_LIST ??= "0"
>> +image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}"
>> +image_package_list() {
>> +    if
>> [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}" =
>> 'True' ]; then
>> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
>> +          -f
>> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n'
>> -W > \
>> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
>> +    fi
>> +}
>> +ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list"
>> +
>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>> index ec6bd39..85bab64 100644
>> --- a/meta/classes/image.bbclass
>> +++ b/meta/classes/image.bbclass
>> @@ -68,6 +68,7 @@ inherit image-tools-extension
>>  inherit image-postproc-extension
>>  inherit image-locales-extension
>>  inherit image-account-extension
>> +inherit image-package-list-extension
>>  
>>  # Extra space for rootfs in MB
>>  ROOTFS_EXTRA ?= "64"
> 
-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] meta/classes: generate bill of material from image

[Baurzhan Ismagulov]

On Mon, Aug 05, 2019 at 04:43:27PM +0200, Henning Schild wrote:
> do you have a real use-case for that? In fact we do have similar things
> internally already but did not propose it yet. Reason is you still need
> to find all the magic switches to collect the source and binary package
> versions, and you most likely want a machine readable format.
> 
> In fact we should just fix that broken offline cache thing and enable
> src-packages for it. That would generate a list that will hopefully be
> complete and in a well defined format.
> 
> Without the real use-case and the post-processing this is just another
> obscure Isar feature and i am against merging it.

Isar has started to be useful today. As such, it has known limitations. E.g.,
bitbake isn't the perfect tool for the job, either. That said, if we waited
till we develop the perfect tool, we wouldn't have been here. If the feature
has value, I wouldn't like to discard it outright.

Regarding the base-apt design -- here we need either debootstrap introspection
(which it possibly doesn't have having been developed in shell to run on
non-Debian systems), or reimplement multistrap in perl (as discussed with other
debian-embedded developers). According to an ELBE developer, they also use this
approach, so this seems to be state of the art ATM. But I agree that solving
this upfront in a separate step would simplify the architecture.

With kind regards,
Baurzhan.

Re: [PATCH] meta/classes: generate bill of material from image

[Henning Schild]

Am Mon, 5 Aug 2019 16:48:36 +0200
schrieb Jan Kiszka <jan.kiszka@siemens.com>:

> On 05.08.19 16:43, [ext] Henning Schild wrote:
> > Hi Quirin,
> > 
> > do you have a real use-case for that? In fact we do have similar
> > things internally already but did not propose it yet. Reason is you
> > still need to find all the magic switches to collect the source and
> > binary package versions, and you most likely want a machine
> > readable format.  
> 
> The use cases are the same what we have that internal stuff for.
> 
> > 
> > In fact we should just fix that broken offline cache thing and
> > enable src-packages for it. That would generate a list that will
> > hopefully be complete and in a well defined format.  
> 
> Good points: The format should be easily parseable.
> 
> > 
> > Without the real use-case and the post-processing this is just
> > another obscure Isar feature and i am against merging it.  
> 
> How would you do it otherwise? Tell the beginners to remember which
> command to run with which switches on which subdirs?
> 
> Quirin and I were also discussing the first extension: We need to
> collect information about additional packages that are not part of
> the rootfs but the image, primarily bootloaders. Providing a
> framework - however that may look like
> - for recipe authors to report them would be valuable and can be
> achieved by any local solution that cleanly.

What i meant to say was. Please contact all the people that are already
doing that and collect the requirements.
Because if we miss any of the known ones, the feature will be an obscure
one.

Henning

> Jan
> 
> > 
> > Henning
> > 
> > Am Mon, 5 Aug 2019 16:07:42 +0200
> > schrieb "[ext] Q. Gylstorff" <Quirin.Gylstorff@siemens.com>:
> >   
> >> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> >>
> >> To create products it is necessary to have a list
> >> of used packages for clearance and to security monitoring.
> >> To get a simple list of packages use dpkg-query and generate
> >> a list with the following pattern:
> >>
> >> source name| source version | binary package name | binary version
> >>
> >> To use it add following line to the local.conf:
> >> ISAR_DO_PACKAGE_LIST ?= "1"
> >>
> >> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> >> ---
> >>  meta-isar/conf/local.conf.sample                 |  4 ++++
> >>  .../classes/image-package-list-extension.bbclass | 16
> >> ++++++++++++++++ meta/classes/image.bbclass                       |
> >> 1 + 3 files changed, 21 insertions(+)
> >>  create mode 100644
> >> meta/classes/image-package-list-extension.bbclass
> >>
> >> diff --git a/meta-isar/conf/local.conf.sample
> >> b/meta-isar/conf/local.conf.sample index 5b3a0a1..d188051 100644
> >> --- a/meta-isar/conf/local.conf.sample
> >> +++ b/meta-isar/conf/local.conf.sample
> >> @@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw
> >> example-module-${KERNEL_NAME} enable-fsc # NOTE: this works on
> >> build host >= stretch for armhf, arm64 and amd64 targets for now.
> >> ISAR_CROSS_COMPILE ?= "0" 
> >> +#
> >> +# Generate package list 
> >> +ISAR_DO_PACKAGE_LIST ?= "1"
> >> +
> >>  #
> >>  # Uncomment this to enable use of cached base repository
> >>  #ISAR_USE_CACHED_BASE_REPO ?= "1"
> >> diff --git a/meta/classes/image-package-list-extension.bbclass
> >> b/meta/classes/image-package-list-extension.bbclass new file mode
> >> 100644 index 0000000..558922e
> >> --- /dev/null
> >> +++ b/meta/classes/image-package-list-extension.bbclass
> >> @@ -0,0 +1,16 @@
> >> +# This software is a part of ISAR.
> >> +# Copyright (C) Siemens AG, 2019
> >> +#
> >> +# SPDX-License-Identifier: MIT
> >> +
> >> +ISAR_DO_PACKAGE_LIST ??= "0"
> >> +image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}"
> >> +image_package_list() {
> >> +    if
> >> [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}"
> >> = 'True' ]; then
> >> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
> >> +          -f
> >> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n'
> >> -W > \
> >> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
> >> +    fi
> >> +}
> >> +ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list"
> >> +
> >> diff --git a/meta/classes/image.bbclass
> >> b/meta/classes/image.bbclass index ec6bd39..85bab64 100644
> >> --- a/meta/classes/image.bbclass
> >> +++ b/meta/classes/image.bbclass
> >> @@ -68,6 +68,7 @@ inherit image-tools-extension
> >>  inherit image-postproc-extension
> >>  inherit image-locales-extension
> >>  inherit image-account-extension
> >> +inherit image-package-list-extension
> >>  
> >>  # Extra space for rootfs in MB
> >>  ROOTFS_EXTRA ?= "64"  
> >   

Re: [PATCH] meta/classes: generate bill of material from image

[Jan Kiszka]

On 05.08.19 17:08, Henning Schild wrote:
> Am Mon, 5 Aug 2019 16:48:36 +0200
> schrieb Jan Kiszka <jan.kiszka@siemens.com>:
> 
>> On 05.08.19 16:43, [ext] Henning Schild wrote:
>>> Hi Quirin,
>>>
>>> do you have a real use-case for that? In fact we do have similar
>>> things internally already but did not propose it yet. Reason is you
>>> still need to find all the magic switches to collect the source and
>>> binary package versions, and you most likely want a machine
>>> readable format.  
>>
>> The use cases are the same what we have that internal stuff for.
>>
>>>
>>> In fact we should just fix that broken offline cache thing and
>>> enable src-packages for it. That would generate a list that will
>>> hopefully be complete and in a well defined format.  
>>
>> Good points: The format should be easily parseable.
>>
>>>
>>> Without the real use-case and the post-processing this is just
>>> another obscure Isar feature and i am against merging it.  
>>
>> How would you do it otherwise? Tell the beginners to remember which
>> command to run with which switches on which subdirs?
>>
>> Quirin and I were also discussing the first extension: We need to
>> collect information about additional packages that are not part of
>> the rootfs but the image, primarily bootloaders. Providing a
>> framework - however that may look like
>> - for recipe authors to report them would be valuable and can be
>> achieved by any local solution that cleanly.
> 
> What i meant to say was. Please contact all the people that are already
> doing that and collect the requirements.
> Because if we miss any of the known ones, the feature will be an obscure
> one.

That's what this is patch proposal is for: get the attention. :)

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Quirin,

On 05/08/2019 16.07, [ext] Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> To use it add following line to the local.conf:
> ISAR_DO_PACKAGE_LIST ?= "1"

I would rather that the commit message would not suggest adding this to 
the local.conf and instead adding this to the image recipe variable 
scope, wherever that is a .bb, .bbclass, .bbappend or any .conf file.

> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
>   meta-isar/conf/local.conf.sample                 |  4 ++++
>   .../classes/image-package-list-extension.bbclass | 16 ++++++++++++++++
>   meta/classes/image.bbclass                       |  1 +
>   3 files changed, 21 insertions(+)
>   create mode 100644 meta/classes/image-package-list-extension.bbclass
> 
> diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
> index 5b3a0a1..d188051 100644
> --- a/meta-isar/conf/local.conf.sample
> +++ b/meta-isar/conf/local.conf.sample
> @@ -170,6 +170,10 @@ IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsc
>   # NOTE: this works on build host >= stretch for armhf, arm64 and amd64 targets for now.
>   ISAR_CROSS_COMPILE ?= "0"
>   
> +#
> +# Generate package list
> +ISAR_DO_PACKAGE_LIST ?= "1"
> +
>   #
>   # Uncomment this to enable use of cached base repository
>   #ISAR_USE_CACHED_BASE_REPO ?= "1"
> diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
> new file mode 100644
> index 0000000..558922e
> --- /dev/null
> +++ b/meta/classes/image-package-list-extension.bbclass
> @@ -0,0 +1,16 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +
> +ISAR_DO_PACKAGE_LIST ??= "0"
> +image_package_list[dirs] = "${DEPLOY_DIR_IMAGE}"
> +image_package_list() {

Its a postprocess function so call it 'image_postprocess_package_list' 
instead.

> +    if [ "${@repr(bb.utils.to_boolean(d.getVar('ISAR_DO_PACKAGE_LIST')))}" = 'True' ]; then

I don't like all those boolean values and tried to establish a better 
method using features. Take a look at the rootfs.bbclass, for example:

   ROOTFS_POSTPROCESS_COMMAND += 
"${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs', 
'rootfs_postprocess_finalize', '', d)}"

You can adapt that. For instance like this:

   ROOTFS_POSTPROCESS_COMMAND += 
"${@bb.utils.contains('ROOTFS_FEATURES', 'generate-package-list', 
'image_postprocess_package_list', '', d)}"

Also it probably makes sense to have this as a default image feature. 
Otherwise people will not know about it and try to implement it themselves.

> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
> +          -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst

As others have said, make it easily machine readable. JSON maybe?

Maybe something like:

   echo "[" > ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
   dpkg-query ... -f '{"source_package": "${source:Package}", 
source_version": "${source:Version}", ...},\n' ... >> 
${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst
   echo "]" >> ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.packages.lst

Also maybe copy the licenses (/usr/share/doc/$package/copyright) 
somewhere? That might conflict with Hennings 'isar-exclude-docs' 
package, though... Maybe that can be changed to move them somewhere 
instead of removing them?

kind regards,
Claudius

> +    fi
> +}
> +ROOTFS_POSTPROCESS_COMMAND =+ "image_package_list"
> +
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..85bab64 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -68,6 +68,7 @@ inherit image-tools-extension
>   inherit image-postproc-extension
>   inherit image-locales-extension
>   inherit image-account-extension
> +inherit image-package-list-extension
>   
>   # Extra space for rootfs in MB
>   ROOTFS_EXTRA ?= "64"
> 

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH] meta/classes: generate bill of material from image

[Baurzhan Ismagulov]

Hello Claudius,

On Tue, Aug 06, 2019 at 10:07:03AM +0200, Claudius Heine wrote:
> I don't like all those boolean values and tried to establish a better method
> using features.

A general question here: What are the advantages of using features compared to
boolean variables? Do you know why they have been introduced?

With kind regards,
Baurzhan.

Re: [PATCH] meta/classes: generate bill of material from image

[Jan Kiszka]

On 06.08.19 10:07, [ext] Claudius Heine wrote:
> Hi Quirin,
> 
> On 05/08/2019 16.07, [ext] Q. Gylstorff wrote:
>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>
>> To create products it is necessary to have a list
>> of used packages for clearance and to security monitoring.
>> To get a simple list of packages use dpkg-query and generate
>> a list with the following pattern:
>>
>> source name| source version | binary package name | binary version
>>
>> To use it add following line to the local.conf:
>> ISAR_DO_PACKAGE_LIST ?= "1"
> 
> I would rather that the commit message would not suggest adding this to the
> local.conf and instead adding this to the image recipe variable scope, wherever
> that is a .bb, .bbclass, .bbappend or any .conf file.

I think the best approach is making this opt-out, generating this list by
default (as in Yocto) and allowing users to disable it here this way.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Baurzahn,

On 06/08/2019 10.36, Baurzhan Ismagulov wrote:
> Hello Claudius,
> 
> On Tue, Aug 06, 2019 at 10:07:03AM +0200, Claudius Heine wrote:
>> I don't like all those boolean values and tried to establish a better method
>> using features.
> 
> A general question here: What are the advantages of using features compared to
> boolean variables?

- easier to document and structure
- easier to look up in bitbake -e
- easier to use in in-line python with bb.utils.contains
- more in-line with the IMAGE_FEATURES, DISTRO_FEATURES, etc. of OE/YP.

> Do you know why they have been introduced?

Boolean variable? Probably because they were introduced because they are 
quick and dirty.

kind regards,
Claudius

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Jan,

On 06/08/2019 10.38, Jan Kiszka wrote:
> On 06.08.19 10:07, [ext] Claudius Heine wrote:
>> Hi Quirin,
>>
>> On 05/08/2019 16.07, [ext] Q. Gylstorff wrote:
>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>
>>> To create products it is necessary to have a list
>>> of used packages for clearance and to security monitoring.
>>> To get a simple list of packages use dpkg-query and generate
>>> a list with the following pattern:
>>>
>>> source name| source version | binary package name | binary version
>>>
>>> To use it add following line to the local.conf:
>>> ISAR_DO_PACKAGE_LIST ?= "1"
>>
>> I would rather that the commit message would not suggest adding this to the
>> local.conf and instead adding this to the image recipe variable scope, wherever
>> that is a .bb, .bbclass, .bbappend or any .conf file.
> 
> I think the best approach is making this opt-out, generating this list by
> default (as in Yocto) and allowing users to disable it here this way.

I agree. As I commented in my review.

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH] meta/classes: generate bill of material from image

[Baurzhan Ismagulov]

On Tue, Aug 06, 2019 at 10:47:09AM +0200, Claudius Heine wrote:
> - easier to document and structure
> - easier to look up in bitbake -e
> - easier to use in in-line python with bb.utils.contains
> - more in-line with the IMAGE_FEATURES, DISTRO_FEATURES, etc. of OE/YP.

Thanks, will have a look.


> > Do you know why they have been introduced?
> 
> Boolean variable?

No, I mean the features?


With kind regards,
Baurzhan.

Re: [PATCH] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Baurzahn,

On 06/08/2019 11.03, Baurzhan Ismagulov wrote:
> On Tue, Aug 06, 2019 at 10:47:09AM +0200, Claudius Heine wrote:
>> - easier to document and structure
>> - easier to look up in bitbake -e
>> - easier to use in in-line python with bb.utils.contains
>> - more in-line with the IMAGE_FEATURES, DISTRO_FEATURES, etc. of OE/YP.
> 
> Thanks, will have a look.
> 
> 
>>> Do you know why they have been introduced?
>>
>> Boolean variable?
> 
> No, I mean the features?

Is that still unclear, or have those listed advantages explained it?

Claudius

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH] meta/classes: generate bill of material from image

[Quirin Gylstorff]

On 8/6/19 10:48 AM, Claudius Heine wrote:
> Hi Jan,
> 
> On 06/08/2019 10.38, Jan Kiszka wrote:
>> On 06.08.19 10:07, [ext] Claudius Heine wrote:
>>> Hi Quirin,
>>>
>>> On 05/08/2019 16.07, [ext] Q. Gylstorff wrote:
>>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>>
>>>> To create products it is necessary to have a list
>>>> of used packages for clearance and to security monitoring.
>>>> To get a simple list of packages use dpkg-query and generate
>>>> a list with the following pattern:
>>>>
>>>> source name| source version | binary package name | binary version
>>>>
>>>> To use it add following line to the local.conf:
>>>> ISAR_DO_PACKAGE_LIST ?= "1"
>>>
>>> I would rather that the commit message would not suggest adding this 
>>> to the
>>> local.conf and instead adding this to the image recipe variable 
>>> scope, wherever
>>> that is a .bb, .bbclass, .bbappend or any .conf file.
>>
>> I think the best approach is making this opt-out, generating this list by
>> default (as in Yocto) and allowing users to disable it here this way.
> 
> I agree. As I commented in my review.
> 
I will change to opt-out that in a v2.

Quirin

[PATCH v2] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
 meta/classes/image-package-list-extension.bbclass | 13 +++++++++++++
 meta/classes/image.bbclass                        |  3 ++-
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/image-package-list-extension.bbclass

diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
new file mode 100644
index 0000000..c746397
--- /dev/null
+++ b/meta/classes/image-package-list-extension.bbclass
@@ -0,0 +1,13 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+
+image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
+image_generate_manifest() {
+      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
+          -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.rootfs.manifest
+}
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
+
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index ec6bd39..60dd9fb 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -58,7 +58,7 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
 
 inherit rootfs
@@ -68,6 +68,7 @@ inherit image-tools-extension
 inherit image-postproc-extension
 inherit image-locales-extension
 inherit image-account-extension
+inherit image-package-list-extension
 
 # Extra space for rootfs in MB
 ROOTFS_EXTRA ?= "64"
-- 
2.20.1

Re: [PATCH v2] meta/classes: generate bill of material from image

[Quirin Gylstorff]

On 8/6/19 3:55 PM, Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest
> 
> Remove the feature with:
> ROOTFS_FEATURES_remove = "generate-manifest"
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
>   meta/classes/image-package-list-extension.bbclass | 13 +++++++++++++
>   meta/classes/image.bbclass                        |  3 ++-
>   2 files changed, 15 insertions(+), 1 deletion(-)
>   create mode 100644 meta/classes/image-package-list-extension.bbclass
> 
> diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
> new file mode 100644
> index 0000000..c746397
> --- /dev/null
> +++ b/meta/classes/image-package-list-extension.bbclass
> @@ -0,0 +1,13 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +
> +image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
> +image_generate_manifest() {
> +      dpkg-query --admindir=${IMAGE_ROOTFS}/var/lib/dpkg/ \
> +          -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
> +          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.rootfs.manifest
> +}
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
> +
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..60dd9fb 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -58,7 +58,7 @@ image_do_mounts() {
>   }
>   
>   ROOTFSDIR = "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
>   ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>   
>   inherit rootfs
> @@ -68,6 +68,7 @@ inherit image-tools-extension
>   inherit image-postproc-extension
>   inherit image-locales-extension
>   inherit image-account-extension
> +inherit image-package-list-extension
>   
>   # Extra space for rootfs in MB
>   ROOTFS_EXTRA ?= "64"
> 

I am thinking about adding json as alternative output format. The yocto 
manifest is:
packagename arch version

The current isar manifest differs from this already.

Kind regards
Quirin

Re: [PATCH v2] meta/classes: generate bill of material from image

[Gernot Hillier]

Hi!

Am 07.08.19 um 09:52 schrieb Quirin Gylstorff:
> 
> 
> On 8/6/19 3:55 PM, Q. Gylstorff wrote:
>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>
>> To create products it is necessary to have a list
>> of used packages for clearance and to security monitoring.
>> To get a simple list of packages use dpkg-query and generate
>> a list with the following pattern:
>>
[...]
> I am thinking about adding json as alternative output format. The yocto 

Why? Is this some established standard somewhere else?

> manifest is:
> packagename arch version
> 
> The current isar manifest differs from this already.

For Debian, the (subtle) differences between source and binary version 
numbers and source and binary package names caused a lot of confusion 
(and effort) in the past, so I'd really love to see both included.

--
Gernot

Re: [PATCH v2] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Gernot,

On 07/08/2019 09.56, [ext] Gernot Hillier wrote:
> Hi!
> 
> Am 07.08.19 um 09:52 schrieb Quirin Gylstorff:
>>
>>
>> On 8/6/19 3:55 PM, Q. Gylstorff wrote:
>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>
>>> To create products it is necessary to have a list
>>> of used packages for clearance and to security monitoring.
>>> To get a simple list of packages use dpkg-query and generate
>>> a list with the following pattern:
>>>
> [...]
>> I am thinking about adding json as alternative output format. The yocto 
> 
> Why? Is this some established standard somewhere else?

I don't know any standard format for manifest files? Do you? I suggested 
JSON because its very easy machine readable and expandable without 
breaking other tools.

We could also go with the Yocto/OE format.

kind regards,
Claudius

> 
>> manifest is:
>> packagename arch version
>>
>> The current isar manifest differs from this already.
> 
> For Debian, the (subtle) differences between source and binary version 
> numbers and source and binary package names caused a lot of confusion 
> (and effort) in the past, so I'd really love to see both included.
> 
> -- 
> Gernot
> 

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH v2] meta/classes: generate bill of material from image

[Gernot Hillier]

Am 07.08.19 um 10:01 schrieb Claudius Heine:
> Hi Gernot,
> 
> On 07/08/2019 09.56, [ext] Gernot Hillier wrote:
>> Hi!
>>
>> Am 07.08.19 um 09:52 schrieb Quirin Gylstorff:
>>>
>>>
>>> On 8/6/19 3:55 PM, Q. Gylstorff wrote:
>>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>>
>>>> To create products it is necessary to have a list
>>>> of used packages for clearance and to security monitoring.
>>>> To get a simple list of packages use dpkg-query and generate
>>>> a list with the following pattern:
>>>>
>> [...]
>>> I am thinking about adding json as alternative output format. The yocto 
>>
>> Why? Is this some established standard somewhere else?
> 
> I don't know any standard format for manifest files? Do you? I suggested 
> JSON because its very easy machine readable and expandable without 
> breaking other tools.

Well, if there's no established standard, I'd prefer to go with 
something which is easy to consume in shell scripts.

If we don't have clear advantages for JSON, I prefer

cut -d "|" -f 1

over

python3 -c "import sys, json; print(json.load(sys.stdin)['name'])"

- or adding another dependency to "jq".

--
Gernot

Re: [PATCH v2] meta/classes: generate bill of material from image

[Claudius Heine]

On 07/08/2019 10.08, Gernot Hillier wrote:
> 
> 
> Am 07.08.19 um 10:01 schrieb Claudius Heine:
>> Hi Gernot,
>>
>> On 07/08/2019 09.56, [ext] Gernot Hillier wrote:
>>> Hi!
>>>
>>> Am 07.08.19 um 09:52 schrieb Quirin Gylstorff:
>>>>
>>>>
>>>> On 8/6/19 3:55 PM, Q. Gylstorff wrote:
>>>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>>>
>>>>> To create products it is necessary to have a list
>>>>> of used packages for clearance and to security monitoring.
>>>>> To get a simple list of packages use dpkg-query and generate
>>>>> a list with the following pattern:
>>>>>
>>> [...]
>>>> I am thinking about adding json as alternative output format. The yocto 
>>>
>>> Why? Is this some established standard somewhere else?
>>
>> I don't know any standard format for manifest files? Do you? I 
>> suggested JSON because its very easy machine readable and expandable 
>> without breaking other tools.
> 
> Well, if there's no established standard, I'd prefer to go with 
> something which is easy to consume in shell scripts.
> 
> If we don't have clear advantages for JSON, I prefer

Well IMO, we do have a clear advantage being JSON is sort of a standard 
format and this is not. But we could go with CSV perhaps?

> 
> cut -d "|" -f 1
> 
> over
> 
> python3 -c "import sys, json; print(json.load(sys.stdin)['name'])"
> 
> - or adding another dependency to "jq".

We don't need an additional dependency in isar. But if you like to use 
json in a shell script, jq might be the way to go.

Otherwise we could also use the debian format and print out the 
installed packages in deb822 [1] format.

regards,
Claudius

[1] http://man7.org/linux/man-pages/man5/deb822.5.html

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH v2] meta/classes: generate bill of material from image

[Gernot Hillier]

Am 07.08.19 um 10:21 schrieb Claudius Heine:
> On 07/08/2019 10.08, Gernot Hillier wrote:
>>
>>
>> Am 07.08.19 um 10:01 schrieb Claudius Heine:
>>> Hi Gernot,
>>>
>>> On 07/08/2019 09.56, [ext] Gernot Hillier wrote:
>>>> Hi!
>>>>
>>>> Am 07.08.19 um 09:52 schrieb Quirin Gylstorff:
>>>>>
>>>>>
>>>>> On 8/6/19 3:55 PM, Q. Gylstorff wrote:
>>>>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>>>>
>>>>>> To create products it is necessary to have a list
>>>>>> of used packages for clearance and to security monitoring.
>>>>>> To get a simple list of packages use dpkg-query and generate
>>>>>> a list with the following pattern:
>>>>>>
>>>> [...]
>>>>> I am thinking about adding json as alternative output format. The 
>>>>> yocto 
>>>>
>>>> Why? Is this some established standard somewhere else?
>>>
>>> I don't know any standard format for manifest files? Do you? I 
>>> suggested JSON because its very easy machine readable and expandable 
>>> without breaking other tools.
>>
>> Well, if there's no established standard, I'd prefer to go with 
>> something which is easy to consume in shell scripts.
>>
>> If we don't have clear advantages for JSON, I prefer
> 
> Well IMO, we do have a clear advantage being JSON is sort of a standard 
> format and this is not. But we could go with CSV perhaps?

Well, this is CSV with "|" as separator. ;)

For sure, I'm fine with whatever separator you like.

> 
>>
>> cut -d "|" -f 1
>>
>> over
>>
>> python3 -c "import sys, json; print(json.load(sys.stdin)['name'])"
>>
>> - or adding another dependency to "jq".
> 
> We don't need an additional dependency in isar. But if you like to use 
> json in a shell script, jq might be the way to go.

The main use case for such manifests is decoupled from the build 
process, i.e. far away from Isar and probably even using such strange 
platforms like Windows. Think about clearing personnel who wants to 
postprocess such lists in their MS Office world

> Otherwise we could also use the debian format and print out the 
> installed packages in deb822 [1] format.

So please don't let us use something which can only be consumed on a 
Debian host.

--
Gernot

Re: [PATCH v2] meta/classes: generate bill of material from image

[Gernot Hillier]

Am 07.08.19 um 10:29 schrieb Gernot Hillier:
>> We don't need an additional dependency in isar. But if you like to use 
>> json in a shell script, jq might be the way to go.
> 
> The main use case for such manifests is decoupled from the build 
> process, i.e. far away from Isar and probably even using such strange 
> platforms like Windows. Think about clearing personnel who wants to 
> postprocess such lists in their MS Office world

And another important point I forgot: those manifests should be easy to 
create on targets/VMs without Isar or any other additional tool.

We will continue to have Debian images which were not created with Isar 
and we can also safely assume that some development teams will manually 
modify Isar images later on.

So we need some format which can be easily created using a simple shell 
command which works on virtually any Debian image.

--
Gernot

Re: [PATCH v2] meta/classes: generate bill of material from image

[Baurzhan Ismagulov]

On Wed, Aug 07, 2019 at 12:00:22PM +0200, Gernot Hillier wrote:
> Am 07.08.19 um 10:29 schrieb Gernot Hillier:
> > > We don't need an additional dependency in isar. But if you like to
> > > use json in a shell script, jq might be the way to go.
> > 
> > The main use case for such manifests is decoupled from the build
> > process, i.e. far away from Isar and probably even using such strange
> > platforms like Windows. Think about clearing personnel who wants to
> > postprocess such lists in their MS Office world
> 
> And another important point I forgot: those manifests should be easy to
> create on targets/VMs without Isar or any other additional tool.
> 
> We will continue to have Debian images which were not created with Isar and
> we can also safely assume that some development teams will manually modify
> Isar images later on.
> 
> So we need some format which can be easily created using a simple shell
> command which works on virtually any Debian image.

I second Gernot's both points. If Isar already implements it differently, I
wouldn't necessarily block the patch since it adds value today, but I think
simple Yocto-like format and standalone generation is the right direction to
go. Other formats could be e.g. optional or additional.

With kind regards,
Baurzhan.

Re: [PATCH v2] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Gernot,

On 07/08/2019 12.00, Gernot Hillier wrote:
> Am 07.08.19 um 10:29 schrieb Gernot Hillier:
>>> We don't need an additional dependency in isar. But if you like to 
>>> use json in a shell script, jq might be the way to go.
>>
>> The main use case for such manifests is decoupled from the build 
>> process, i.e. far away from Isar and probably even using such strange 
>> platforms like Windows. Think about clearing personnel who wants to 
>> postprocess such lists in their MS Office world
> 
> And another important point I forgot: those manifests should be easy to 
> create on targets/VMs without Isar or any other additional tool.
> 
> We will continue to have Debian images which were not created with Isar 
> and we can also safely assume that some development teams will manually 
> modify Isar images later on.
> 
> So we need some format which can be easily created using a simple shell 
> command which works on virtually any Debian image.

Ok, those are points I haven't considered. That would point to a more 
common CSV format with '[,;\t ]' separators. This can easily imported in 
MS Office 180.

This script currently uses the dpkg-query of the build system, not the 
one from the buildchroot or image root fs, so the file cannot be created 
easily on other non-debian systems. @Quirin: Have you tried using the 
dpkg-query of the buildchroot?

regards,
Claudius

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH v2] meta/classes: generate bill of material from image

[Quirin Gylstorff]

On 8/7/19 1:27 PM, Claudius Heine wrote:
> Ok, those are points I haven't considered. That would point to a more 
> common CSV format with '[,;\t ]' separators. This can easily imported in 
> MS Office 180.
> 
> This script currently uses the dpkg-query of the build system, not the 
> one from the buildchroot or image root fs, so the file cannot be created 
> easily on other non-debian systems. @Quirin: Have you tried using the 
> dpkg-query of the buildchroot?

No I didn't try the dpkg-query of the buildchroot. I will test it.

> 
> regards,
> Claudius

[PATCH v3] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
Changes:
v3:
Add list of manifest for buildchroot manifest
This list can be exdent to add additional output generators 
v2:
use FEATURE instead of own variable

.../image-package-list-extension.bbclass      | 66 +++++++++++++++++++
 meta/classes/image.bbclass                    |  3 +-
 2 files changed, 68 insertions(+), 1 deletion(-)
 create mode 100644 meta/classes/image-package-list-extension.bbclass

diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
new file mode 100644
index 0000000..11896f1
--- /dev/null
+++ b/meta/classes/image-package-list-extension.bbclass
@@ -0,0 +1,66 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+
+MANIFESTS ?= "target build"
+MANIFEST_build[rootfs] ?= "/var/lib/dpkg"
+MANIFEST_target[rootfs] ?= "${PP_ROOTFS}/var/lib/dpkg"
+
+def gen_manifests_array(d, listname, entryname, flags, verb_flags=None):
+    from itertools import chain
+
+    entries = (d.getVar(listname, True) or "").split()
+    return " ".join(
+        ":".join(
+            chain(
+                (entry,),
+                (
+                    (",".join(
+                        (
+                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+                        ).split()
+                    ) if flag not in (verb_flags or []) else (
+                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+                    )).replace(":","=")
+                    for flag in flags
+                ),
+            )
+        )
+        for entry in entries
+    )
+
+
+IMAGE_MANIFESTS =+ "${@gen_manifests_array(d, 'MANIFESTS', 'MANIFEST', ['rootfs'])}"
+
+image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
+image_generate_manifest() {
+    image_do_mounts
+    list='${@" ".join(d.getVar('IMAGE_MANIFESTS', True).split())} '
+    while true; do
+        list_rest="${list#*:* }"
+        entry="${list%%${list_rest}}"
+        list="${list_rest}"
+
+        if [ -z "${entry}" ]; then
+            break
+        fi
+        # Add colon to the end of the entry and remove trailing space:
+        entry="${entry% }:"
+
+        # Decode entries:
+        name="${entry%%:*}"
+        entry="${entry#${name}:}"
+
+        rootfs="${entry%%:*}"
+        entry="${entry#${rootfs}:}"
+
+        
+        sudo -E chroot ${BUILDCHROOT_DIR} \
+        /usr/bin/dpkg-query --admindir="$rootfs" \
+        -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+        ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
+    done 
+}
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
+
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index ec6bd39..60dd9fb 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -58,7 +58,7 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
 
 inherit rootfs
@@ -68,6 +68,7 @@ inherit image-tools-extension
 inherit image-postproc-extension
 inherit image-locales-extension
 inherit image-account-extension
+inherit image-package-list-extension
 
 # Extra space for rootfs in MB
 ROOTFS_EXTRA ?= "64"
-- 
2.20.1

Re: [PATCH v3] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Quirin,

On 09/08/2019 12.30, [ext] Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest
> 
> Remove the feature with:
> ROOTFS_FEATURES_remove = "generate-manifest"
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
> Changes:
> v3:
> Add list of manifest for buildchroot manifest
> This list can be exdent to add additional output generators
> v2:
> use FEATURE instead of own variable
> 
> .../image-package-list-extension.bbclass      | 66 +++++++++++++++++++
>   meta/classes/image.bbclass                    |  3 +-
>   2 files changed, 68 insertions(+), 1 deletion(-)
>   create mode 100644 meta/classes/image-package-list-extension.bbclass
> 
> diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
> new file mode 100644
> index 0000000..11896f1
> --- /dev/null
> +++ b/meta/classes/image-package-list-extension.bbclass
> @@ -0,0 +1,66 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +
> +MANIFESTS ?= "target build"
> +MANIFEST_build[rootfs] ?= "/var/lib/dpkg"
> +MANIFEST_target[rootfs] ?= "${PP_ROOTFS}/var/lib/dpkg"

Have you planned additional flags for this?

Currently I think that this mechanism is a bit of an overkill for just 
two variables. But since you touched this now and are the second users 
of this, I have further comments ;).

> +
> +def gen_manifests_array(d, listname, entryname, flags, verb_flags=None):
> +    from itertools import chain
> +
> +    entries = (d.getVar(listname, True) or "").split()
> +    return " ".join(
> +        ":".join(
> +            chain(
> +                (entry,),
> +                (
> +                    (",".join(
> +                        (
> +                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> +                        ).split()
> +                    ) if flag not in (verb_flags or []) else (
> +                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> +                    )).replace(":","=")
> +                    for flag in flags
> +                ),
> +            )
> +        )
> +        for entry in entries

Is this a 1:1 copy from image-account-extension.bbclass? If so, then can 
you put those implementations together to avoid code duplication?

I also have to say, I am not 100% happy with that code. While it is very 
efficient, it might be not that maintainable compared to simple loops 
that build the output string using string operators or StringIO. Those 
are probably easier to understand for people not that fluent in python.

> +    )
> +
> +
> +IMAGE_MANIFESTS =+ "${@gen_manifests_array(d, 'MANIFESTS', 'MANIFEST', ['rootfs'])}"
> +
> +image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
> +image_generate_manifest() {
> +    image_do_mounts
> +    list='${@" ".join(d.getVar('IMAGE_MANIFESTS', True).split())} '
> +    while true; do
> +        list_rest="${list#*:* }"
> +        entry="${list%%${list_rest}}"
> +        list="${list_rest}"
> +
> +        if [ -z "${entry}" ]; then
> +            break
> +        fi
> +        # Add colon to the end of the entry and remove trailing space:
> +        entry="${entry% }:"
> +
> +        # Decode entries:
> +        name="${entry%%:*}"
> +        entry="${entry#${name}:}"
> +
> +        rootfs="${entry%%:*}"
> +        entry="${entry#${rootfs}:}"

I also see here some ways to avoid code duplication. Maybe generate this 
shell code in a python function?

Something like:

     ${@gen_shell_list_processing('IMAGE_MANIFESTS', ['name', 'rootfs'], """
         sudo -E chroot ${BUILDCHROOT_DIR} \
         /usr/bin/dpkg-query --admindir="$rootfs" \
         -f 
'${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W\
         ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
""")}

Not sure about the shell part. BB seems to not handle line breaks in 
inline python very well.

To move it outside of the inline would also be possible:

     ${@gen_shell_list_processing_start('IMAGE_MANIFESTS', ['name', 
'rootfs'])}
         sudo -E chroot ${BUILDCHROOT_DIR} \
         /usr/bin/dpkg-query --admindir="$rootfs" \
         -f 
'${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W\
         ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
     ${@gen_shell_list_processing_end()}

gen_shell_list_processing_end would just return 'done;' but it makes is 
more flexible in the future... maybe, IDK. But having 'done' directly 
there would also be ok.


> +
> +
> +        sudo -E chroot ${BUILDCHROOT_DIR} \
> +        /usr/bin/dpkg-query --admindir="$rootfs" \
> +        -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
> +        ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest

Does this also work if you just specify 'dpkg-query' without '/usr/bin/'?

Cheers,
Claudius

> +    done
> +}
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
> +
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..60dd9fb 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -58,7 +58,7 @@ image_do_mounts() {
>   }
>   
>   ROOTFSDIR = "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
>   ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>   
>   inherit rootfs
> @@ -68,6 +68,7 @@ inherit image-tools-extension
>   inherit image-postproc-extension
>   inherit image-locales-extension
>   inherit image-account-extension
> +inherit image-package-list-extension
>   
>   # Extra space for rootfs in MB
>   ROOTFS_EXTRA ?= "64"
> 

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

Re: [PATCH v3] meta/classes: generate bill of material from image

[Quirin Gylstorff]

On 8/12/19 10:04 AM, Claudius Heine wrote:
> Hi Quirin,
> 
> On 09/08/2019 12.30, [ext] Q. Gylstorff wrote:
>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>
>> To create products it is necessary to have a list
>> of used packages for clearance and to security monitoring.
>> To get a simple list of packages use dpkg-query and generate
>> a list with the following pattern:
>>
>> source name| source version | binary package name | binary version
>>
>> The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest
>>
>> Remove the feature with:
>> ROOTFS_FEATURES_remove = "generate-manifest"
>>
>> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>> ---
>> Changes:
>> v3:
>> Add list of manifest for buildchroot manifest
>> This list can be exdent to add additional output generators
>> v2:
>> use FEATURE instead of own variable
>>
>> .../image-package-list-extension.bbclass      | 66 +++++++++++++++++++
>>   meta/classes/image.bbclass                    |  3 +-
>>   2 files changed, 68 insertions(+), 1 deletion(-)
>>   create mode 100644 meta/classes/image-package-list-extension.bbclass
>>
>> diff --git a/meta/classes/image-package-list-extension.bbclass 
>> b/meta/classes/image-package-list-extension.bbclass
>> new file mode 100644
>> index 0000000..11896f1
>> --- /dev/null
>> +++ b/meta/classes/image-package-list-extension.bbclass
>> @@ -0,0 +1,66 @@
>> +# This software is a part of ISAR.
>> +# Copyright (C) Siemens AG, 2019
>> +#
>> +# SPDX-License-Identifier: MIT
>> +
>> +MANIFESTS ?= "target build"
>> +MANIFEST_build[rootfs] ?= "/var/lib/dpkg"
>> +MANIFEST_target[rootfs] ?= "${PP_ROOTFS}/var/lib/dpkg"
> 
> Have you planned additional flags for this?
> 
> Currently I think that this mechanism is a bit of an overkill for just 
> two variables. But since you touched this now and are the second users 
> of this, I have further comments ;).
> 

One Idea was to add use this to add additional generators. If this is 
not a use case anymore than it is overkill.

>> +
>> +def gen_manifests_array(d, listname, entryname, flags, verb_flags=None):
>> +    from itertools import chain
>> +
>> +    entries = (d.getVar(listname, True) or "").split()
>> +    return " ".join(
>> +        ":".join(
>> +            chain(
>> +                (entry,),
>> +                (
>> +                    (",".join(
>> +                        (
>> +                            d.getVarFlag(entryname + "_" + entry, 
>> flag, True) or ""
>> +                        ).split()
>> +                    ) if flag not in (verb_flags or []) else (
>> +                        d.getVarFlag(entryname + "_" + entry, flag, 
>> True) or ""
>> +                    )).replace(":","=")
>> +                    for flag in flags
>> +                ),
>> +            )
>> +        )
>> +        for entry in entries
> 
> Is this a 1:1 copy from image-account-extension.bbclass? If so, then can 
> you put those implementations together to avoid code duplication?
> 
> I also have to say, I am not 100% happy with that code. While it is very 
> efficient, it might be not that maintainable compared to simple loops 
> that build the output string using string operators or StringIO. Those 
> are probably easier to understand for people not that fluent in python.
> 
>> +    )
>> +
>> +
>> +IMAGE_MANIFESTS =+ "${@gen_manifests_array(d, 'MANIFESTS', 
>> 'MANIFEST', ['rootfs'])}"
>> +
>> +image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
>> +image_generate_manifest() {
>> +    image_do_mounts
>> +    list='${@" ".join(d.getVar('IMAGE_MANIFESTS', True).split())} '
>> +    while true; do
>> +        list_rest="${list#*:* }"
>> +        entry="${list%%${list_rest}}"
>> +        list="${list_rest}"
>> +
>> +        if [ -z "${entry}" ]; then
>> +            break
>> +        fi
>> +        # Add colon to the end of the entry and remove trailing space:
>> +        entry="${entry% }:"
>> +
>> +        # Decode entries:
>> +        name="${entry%%:*}"
>> +        entry="${entry#${name}:}"
>> +
>> +        rootfs="${entry%%:*}"
>> +        entry="${entry#${rootfs}:}"
> 
> I also see here some ways to avoid code duplication. Maybe generate this 
> shell code in a python function >
> Something like:
> 
>      ${@gen_shell_list_processing('IMAGE_MANIFESTS', ['name', 'rootfs'], 
> """
>          sudo -E chroot ${BUILDCHROOT_DIR} \
>          /usr/bin/dpkg-query --admindir="$rootfs" \
>          -f 
> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W\
>          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
> """)}
> 
> Not sure about the shell part. BB seems to not handle line breaks in 
> inline python very well.
> 
> To move it outside of the inline would also be possible:
> 
>      ${@gen_shell_list_processing_start('IMAGE_MANIFESTS', ['name', 
> 'rootfs'])}
>          sudo -E chroot ${BUILDCHROOT_DIR} \
>          /usr/bin/dpkg-query --admindir="$rootfs" \
>          -f 
> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W\
>          ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
>      ${@gen_shell_list_processing_end()}
> 
> gen_shell_list_processing_end would just return 'done;' but it makes is 
> more flexible in the future... maybe, IDK. But having 'done' directly 
> there would also be ok.
> 
>

I will test it

>> +
>> +
>> +        sudo -E chroot ${BUILDCHROOT_DIR} \
>> +        /usr/bin/dpkg-query --admindir="$rootfs" \
>> +        -f 
>> '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' 
>> -W > \
>> +        ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
> 
> Does this also work if you just specify 'dpkg-query' without '/usr/bin/'?
> 
> Cheers,
> Claudius
> 
>> +    done
>> +}
>> +ROOTFS_POSTPROCESS_COMMAND += 
>> "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 
>> 'image_generate_manifest', '', d)}"
>> +
>> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
>> index ec6bd39..60dd9fb 100644
>> --- a/meta/classes/image.bbclass
>> +++ b/meta/classes/image.bbclass
>> @@ -58,7 +58,7 @@ image_do_mounts() {
>>   }
>>   ROOTFSDIR = "${IMAGE_ROOTFS}"
>> -ROOTFS_FEATURES += "copy-package-cache clean-package-cache 
>> finalize-rootfs"
>> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache 
>> finalize-rootfs generate-manifest"
>>   ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>>   inherit rootfs
>> @@ -68,6 +68,7 @@ inherit image-tools-extension
>>   inherit image-postproc-extension
>>   inherit image-locales-extension
>>   inherit image-account-extension
>> +inherit image-package-list-extension
>>   # Extra space for rootfs in MB
>>   ROOTFS_EXTRA ?= "64"
>>
> 

Regards
Quirin

Re: [PATCH v3] meta/classes: generate bill of material from image

[Claudius Heine]

On 12/08/2019 11.09, Quirin Gylstorff wrote:
> 
> 
> On 8/12/19 10:04 AM, Claudius Heine wrote:
>> Hi Quirin,
>>
>> On 09/08/2019 12.30, [ext] Q. Gylstorff wrote:
>>> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>>
>>> To create products it is necessary to have a list
>>> of used packages for clearance and to security monitoring.
>>> To get a simple list of packages use dpkg-query and generate
>>> a list with the following pattern:
>>>
>>> source name| source version | binary package name | binary version
>>>
>>> The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest
>>>
>>> Remove the feature with:
>>> ROOTFS_FEATURES_remove = "generate-manifest"
>>>
>>> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
>>> ---
>>> Changes:
>>> v3:
>>> Add list of manifest for buildchroot manifest
>>> This list can be exdent to add additional output generators
>>> v2:
>>> use FEATURE instead of own variable
>>>
>>> .../image-package-list-extension.bbclass      | 66 +++++++++++++++++++
>>>   meta/classes/image.bbclass                    |  3 +-
>>>   2 files changed, 68 insertions(+), 1 deletion(-)
>>>   create mode 100644 meta/classes/image-package-list-extension.bbclass
>>>
>>> diff --git a/meta/classes/image-package-list-extension.bbclass 
>>> b/meta/classes/image-package-list-extension.bbclass
>>> new file mode 100644
>>> index 0000000..11896f1
>>> --- /dev/null
>>> +++ b/meta/classes/image-package-list-extension.bbclass
>>> @@ -0,0 +1,66 @@
>>> +# This software is a part of ISAR.
>>> +# Copyright (C) Siemens AG, 2019
>>> +#
>>> +# SPDX-License-Identifier: MIT
>>> +
>>> +MANIFESTS ?= "target build"
>>> +MANIFEST_build[rootfs] ?= "/var/lib/dpkg"
>>> +MANIFEST_target[rootfs] ?= "${PP_ROOTFS}/var/lib/dpkg"
>>
>> Have you planned additional flags for this?
>>
>> Currently I think that this mechanism is a bit of an overkill for just 
>> two variables. But since you touched this now and are the second users 
>> of this, I have further comments ;).
>>
> 
> One Idea was to add use this to add additional generators. If this is 
> not a use case anymore than it is overkill.

 From my perspective, having multiple manifest generators in upstream is 
not a use-case anyway. I would like just one that covers most of the 
common use-cases. What those 'common use-cases' are should probably be 
documented somewhere. If CSV fits the bill of those, then I am fine with it.

I just suggested JSON because that might be easier to integrate with 
tools I am used to. But I am probably not the person using the manifest 
information anyway.

The problem with arguing about the output format is probably that there 
is no clear listing of all the use-cases. For instance Gernots remark 
that the manifest should be directly importable in Excel was new to me, 
but makes sense. I just thought before that the manifest file is further 
processed by scripts to generate some human readable documents or 
integrate it into a web app or database. In those cases a structured 
text serialization format might have been better.

cheers,
Claudius

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

[PATCH v4] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
Changes:
v4:
Add sdk rootfs to manifest
Avoid duplicated code and move gen_accounts_array and gen_manifest_array to
  shell-list-processing-helper
call dpkg-query from $PATH

v3:
Add list of manifest for buildchroot manifest
This list can be exdent to add additional output generators 

v2:
use FEATURE instead of own variable

 meta/classes/image-account-extension.bbclass  | 28 ++--------
 .../image-package-list-extension.bbclass      | 54 +++++++++++++++++++
 meta/classes/image.bbclass                    |  3 +-
 .../shell-list-processing-helper.bbclass      | 30 +++++++++++
 4 files changed, 89 insertions(+), 26 deletions(-)
 create mode 100644 meta/classes/image-package-list-extension.bbclass
 create mode 100644 meta/classes/shell-list-processing-helper.bbclass

diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
index 22754da..df44c49 100644
--- a/meta/classes/image-account-extension.bbclass
+++ b/meta/classes/image-account-extension.bbclass
@@ -25,36 +25,14 @@ GROUPS ??= ""
 #GROUP_root[gid] = ""
 #GROUP_root[flags] = "system"
 
-def gen_accounts_array(d, listname, entryname, flags, verb_flags=None):
-    from itertools import chain
-
-    entries = (d.getVar(listname, True) or "").split()
-    return " ".join(
-        ":".join(
-            chain(
-                (entry,),
-                (
-                    (",".join(
-                        (
-                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
-                        ).split()
-                    ) if flag not in (verb_flags or []) else (
-                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
-                    )).replace(":","=")
-                    for flag in flags
-                ),
-            )
-        )
-        for entry in entries
-    )
-
+inherit shell-list-processing-helper
 # List of space separated entries, where each entry has the format:
 # username:encryptedpassword:expiredate:inactivenumber:userid:groupid:comment:homedir:shell:group1,group2:flag1,flag2
-IMAGE_ACCOUNTS_USERS =+ "${@gen_accounts_array(d, 'USERS', 'USER', ['password',  'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
+IMAGE_ACCOUNTS_USERS =+ "${@gen_shell_list(d, 'USERS', 'USER', ['password',  'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
 
 # List of space separated entries, where each entry has the format:
 # groupname:groupid:flag1,flag2
-IMAGE_ACCOUNTS_GROUPS =+ "${@gen_accounts_array(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
+IMAGE_ACCOUNTS_GROUPS =+ "${@gen_shell_list(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
 
 ROOTFS_CONFIGURE_COMMAND += "image_configure_accounts"
 image_configure_accounts[weight] = "3"
diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
new file mode 100644
index 0000000..0aa3015
--- /dev/null
+++ b/meta/classes/image-package-list-extension.bbclass
@@ -0,0 +1,54 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+MANIFESTS ?= "target build sdk"
+DPKG_DIR ?= "/var/lib/dpkg"
+# rootfs needs to be mounted inside of buildchroot
+MANIFEST_build[rootfs] ?= "${DPKG_DIR}"
+MANIFEST_target[rootfs] ?= "${PP_ROOTFS}${DPKG_DIR}"
+MANIFEST_sdk[rootfs] ?= "/work/${DISTRO}-${DISTRO_ARCH}/sdkchroot-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}/rootfs${DPKG_DIR}"
+
+inherit shell-list-processing-helper 
+IMAGE_MANIFESTS =+ "${@gen_shell_list(d, 'MANIFESTS', 'MANIFEST', ['rootfs'])}"
+
+do_image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
+image_generate_manifest() {
+    image_do_mounts
+    # mount working directory to access sdk rootfs
+    sudo -s <<'EOSUDO'
+        ( flock 9
+            mkdir -p ${BUILDCHROOT_DIR}/work
+            if ! mountpoint ${BUILDCHROOT_DIR}/work >/dev/null 2>&1; then
+                mount --bind --make-private ${TMPDIR}/work ${BUILDCHROOT_DIR}/work
+            fi
+        ) 9>${MOUNT_LOCKFILE}
+EOSUDO
+    list='${@" ".join(d.getVar('IMAGE_MANIFESTS', True).split())} '
+    while true; do
+        list_rest="${list#*:* }"
+        entry="${list%%${list_rest}}"
+        list="${list_rest}"
+
+        if [ -z "${entry}" ]; then
+            break
+        fi
+        # Add colon to the end of the entry and remove trailing space:
+        entry="${entry% }:"
+
+        # Decode entries:
+        name="${entry%%:*}"
+        entry="${entry#${name}:}"
+
+        rootfs="${entry%%:*}"
+        entry="${entry#${rootfs}:}"
+        if sudo -E chroot ${BUILDCHROOT_DIR} test -d "$rootfs"; then 
+        sudo -E chroot ${BUILDCHROOT_DIR} \
+        dpkg-query --admindir="$rootfs" \
+        -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+        ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
+        fi
+    done
+}
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
+
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index ec6bd39..60dd9fb 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -58,7 +58,7 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
 
 inherit rootfs
@@ -68,6 +68,7 @@ inherit image-tools-extension
 inherit image-postproc-extension
 inherit image-locales-extension
 inherit image-account-extension
+inherit image-package-list-extension
 
 # Extra space for rootfs in MB
 ROOTFS_EXTRA ?= "64"
diff --git a/meta/classes/shell-list-processing-helper.bbclass b/meta/classes/shell-list-processing-helper.bbclass
new file mode 100644
index 0000000..105066b
--- /dev/null
+++ b/meta/classes/shell-list-processing-helper.bbclass
@@ -0,0 +1,30 @@
+# This software is a part of ISAR.
+# Copyright (C) Siemens AG, 2019
+#
+# SPDX-License-Identifier: MIT
+#
+# This class extends the image.bbclass for creating user accounts and groups.
+
+def gen_shell_list(d, listname, entryname, flags, verb_flags=None):
+    from itertools import chain
+
+    entries = (d.getVar(listname, True) or "").split()
+    return " ".join(
+        ":".join(
+            chain(
+                (entry,),
+                (
+                    (",".join(
+                        (
+                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+                        ).split()
+                    ) if flag not in (verb_flags or []) else (
+                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
+                    )).replace(":","=")
+                    for flag in flags
+                ),
+            )
+        )
+        for entry in entries
+    )
+
-- 
2.20.1

Re: [PATCH v4] meta/classes: generate bill of material from image

[Claudius Heine]

Hi Quirin,

On 13/08/2019 10.18, [ext] Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> The list is stored in ${IMAGE_FULLNAME}.rootfs.manifest
> 
> Remove the feature with:
> ROOTFS_FEATURES_remove = "generate-manifest"
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
> Changes:
> v4:
> Add sdk rootfs to manifest
> Avoid duplicated code and move gen_accounts_array and gen_manifest_array to
>    shell-list-processing-helper
> call dpkg-query from $PATH
> 
> v3:
> Add list of manifest for buildchroot manifest
> This list can be exdent to add additional output generators
> 
> v2:
> use FEATURE instead of own variable
> 
>   meta/classes/image-account-extension.bbclass  | 28 ++--------
>   .../image-package-list-extension.bbclass      | 54 +++++++++++++++++++
>   meta/classes/image.bbclass                    |  3 +-
>   .../shell-list-processing-helper.bbclass      | 30 +++++++++++
>   4 files changed, 89 insertions(+), 26 deletions(-)
>   create mode 100644 meta/classes/image-package-list-extension.bbclass
>   create mode 100644 meta/classes/shell-list-processing-helper.bbclass
> 
> diff --git a/meta/classes/image-account-extension.bbclass b/meta/classes/image-account-extension.bbclass
> index 22754da..df44c49 100644
> --- a/meta/classes/image-account-extension.bbclass
> +++ b/meta/classes/image-account-extension.bbclass
> @@ -25,36 +25,14 @@ GROUPS ??= ""
>   #GROUP_root[gid] = ""
>   #GROUP_root[flags] = "system"
>   
> -def gen_accounts_array(d, listname, entryname, flags, verb_flags=None):
> -    from itertools import chain
> -
> -    entries = (d.getVar(listname, True) or "").split()
> -    return " ".join(
> -        ":".join(
> -            chain(
> -                (entry,),
> -                (
> -                    (",".join(
> -                        (
> -                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> -                        ).split()
> -                    ) if flag not in (verb_flags or []) else (
> -                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> -                    )).replace(":","=")
> -                    for flag in flags
> -                ),
> -            )
> -        )
> -        for entry in entries
> -    )
> -
> +inherit shell-list-processing-helper
>   # List of space separated entries, where each entry has the format:
>   # username:encryptedpassword:expiredate:inactivenumber:userid:groupid:comment:homedir:shell:group1,group2:flag1,flag2
> -IMAGE_ACCOUNTS_USERS =+ "${@gen_accounts_array(d, 'USERS', 'USER', ['password',  'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
> +IMAGE_ACCOUNTS_USERS =+ "${@gen_shell_list(d, 'USERS', 'USER', ['password',  'expire', 'inactive', 'uid', 'gid', 'comment', 'home', 'shell', 'groups', 'flags'], ['password', 'comment', 'home', 'shell'])}"
>   
>   # List of space separated entries, where each entry has the format:
>   # groupname:groupid:flag1,flag2
> -IMAGE_ACCOUNTS_GROUPS =+ "${@gen_accounts_array(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
> +IMAGE_ACCOUNTS_GROUPS =+ "${@gen_shell_list(d, 'GROUPS', 'GROUP', ['gid', 'flags'])}"
>   
>   ROOTFS_CONFIGURE_COMMAND += "image_configure_accounts"
>   image_configure_accounts[weight] = "3"
> diff --git a/meta/classes/image-package-list-extension.bbclass b/meta/classes/image-package-list-extension.bbclass
> new file mode 100644
> index 0000000..0aa3015
> --- /dev/null
> +++ b/meta/classes/image-package-list-extension.bbclass
> @@ -0,0 +1,54 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +MANIFESTS ?= "target build sdk"
> +DPKG_DIR ?= "/var/lib/dpkg"
> +# rootfs needs to be mounted inside of buildchroot
> +MANIFEST_build[rootfs] ?= "${DPKG_DIR}"
> +MANIFEST_target[rootfs] ?= "${PP_ROOTFS}${DPKG_DIR}"
> +MANIFEST_sdk[rootfs] ?= "/work/${DISTRO}-${DISTRO_ARCH}/sdkchroot-${HOST_DISTRO}-${HOST_ARCH}-${DISTRO_ARCH}/rootfs${DPKG_DIR}"

That is a long and pretty explicit path. Can you change this to use more 
commonly used variables and if those are missing define them somewhere 
global where sdkchroot and others can use them?

Otherwise if those paths are changed at some point in the future, we 
have to hunt down every of those magic variables to fix them as well, 
instead of just at one global point.

You should also probably write something about the user facing interface 
of this class in the documentation.

> +
> +inherit shell-list-processing-helper
> +IMAGE_MANIFESTS =+ "${@gen_shell_list(d, 'MANIFESTS', 'MANIFEST', ['rootfs'])}"
> +
> +do_image_generate_manifest[dirs] = "${DEPLOY_DIR_IMAGE}"
> +image_generate_manifest() {
> +    image_do_mounts
> +    # mount working directory to access sdk rootfs
> +    sudo -s <<'EOSUDO'
> +        ( flock 9
> +            mkdir -p ${BUILDCHROOT_DIR}/work
> +            if ! mountpoint ${BUILDCHROOT_DIR}/work >/dev/null 2>&1; then
> +                mount --bind --make-private ${TMPDIR}/work ${BUILDCHROOT_DIR}/work
> +            fi

Does that mean that 'do_image_generate_manifest' now depends on 
sdkchroot? Is that task dependency missing?

Personally I would have the manifest for sdkchroot 'opt-in', since I 
don't want to create a sdkchroot just to build a image everytime.

> +        ) 9>${MOUNT_LOCKFILE}
> +EOSUDO
> +    list='${@" ".join(d.getVar('IMAGE_MANIFESTS', True).split())} '
> +    while true; do
> +        list_rest="${list#*:* }"
> +        entry="${list%%${list_rest}}"
> +        list="${list_rest}"
> +
> +        if [ -z "${entry}" ]; then
> +            break
> +        fi
> +        # Add colon to the end of the entry and remove trailing space:
> +        entry="${entry% }:"
> +
> +        # Decode entries:
> +        name="${entry%%:*}"
> +        entry="${entry#${name}:}"
> +
> +        rootfs="${entry%%:*}"
> +        entry="${entry#${rootfs}:}"

Maybe put a empty line here. You had 3 here before, now none, just 
settle with one, maybe two, if you are feeling generous today. :)

> +        if sudo -E chroot ${BUILDCHROOT_DIR} test -d "$rootfs"; then
> +        sudo -E chroot ${BUILDCHROOT_DIR} \
> +        dpkg-query --admindir="$rootfs" \
> +        -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
> +        ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}."$name".manifest
> +        fi
> +    done
> +}
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'image_generate_manifest', '', d)}"
> +
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..60dd9fb 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -58,7 +58,7 @@ image_do_mounts() {
>   }
>   
>   ROOTFSDIR = "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
>   ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>   
>   inherit rootfs
> @@ -68,6 +68,7 @@ inherit image-tools-extension
>   inherit image-postproc-extension
>   inherit image-locales-extension
>   inherit image-account-extension
> +inherit image-package-list-extension
>   
>   # Extra space for rootfs in MB
>   ROOTFS_EXTRA ?= "64"
> diff --git a/meta/classes/shell-list-processing-helper.bbclass b/meta/classes/shell-list-processing-helper.bbclass
> new file mode 100644
> index 0000000..105066b
> --- /dev/null
> +++ b/meta/classes/shell-list-processing-helper.bbclass
> @@ -0,0 +1,30 @@
> +# This software is a part of ISAR.
> +# Copyright (C) Siemens AG, 2019
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +# This class extends the image.bbclass for creating user accounts and groups.

To much copy pasta.

> +
> +def gen_shell_list(d, listname, entryname, flags, verb_flags=None):
> +    from itertools import chain
> +
> +    entries = (d.getVar(listname, True) or "").split()
> +    return " ".join(
> +        ":".join(
> +            chain(
> +                (entry,),
> +                (
> +                    (",".join(
> +                        (
> +                            d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> +                        ).split()
> +                    ) if flag not in (verb_flags or []) else (
> +                        d.getVarFlag(entryname + "_" + entry, flag, True) or ""
> +                    )).replace(":","=")
> +                    for flag in flags
> +                ),
> +            )
> +        )
> +        for entry in entries
> +    )
> +

I a still a bit unconvinced that we need the list feature here. Since 
that feature is more for the usecase that there needs to be a flexible 
number of configuration items each with multiple parameters. Like users 
and groups. That needs to be customized by the end user.

Here we now have 3, with each just one setting each.

Instead I would think that this could be a 'rootfs' extension, where 
each rootfs (every image, buildchroot and sdk) can create their own 
package listing.

regards,
Claudius

-- 
DENX Software Engineering GmbH,      Managing Director: Wolfgang Denk
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-54 Fax: (+49)-8142-66989-80 Email: ch@denx.de

[PATCH v5] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

All rootfs generate the list by default. Currently the f
following lists are generated:
- buildchroot-${DISTRO}-${ARCH}.manifest
- ${IMAGE}-${DISTRO}-${ARCH}.manifest
- optional: sdkchroot-${DISTRO}-${ARCH}.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
 meta/classes/image.bbclass  |  2 +-
 meta/classes/rootfs.bbclass | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index ec6bd39..70c46d2 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -58,7 +58,7 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
 
 inherit rootfs
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index c7e0435..59edcde 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -10,7 +10,8 @@ ROOTFS_PACKAGES ?= ""
 # 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
 # 'clean-package-cache' - delete package cache from rootfs
 # 'finalize-rootfs' - delete files needed to chroot into the rootfs
-ROOTFS_FEATURES ?= ""
+# 'generate-manifest' - generate a package manifest of the rootfs into ${IMAGE_DEPLOY_DIR}
+ROOTFS_FEATURES ?= "generate-manifest"
 
 ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
 
@@ -212,6 +213,15 @@ rootfs_postprocess_finalize() {
 EOSUDO
 }
 
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'rootfs_generate_manifest', '', d)}"
+rootfs_generate_manifest () {
+    mkdir -p ${DEPLOY_DIR_IMAGE}
+    sudo -E chroot '${ROOTFSDIR}' \
+    dpkg-query \
+    -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+    ${DEPLOY_DIR_IMAGE}/"${PF}".manifest
+}
+
 do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
 python do_rootfs_postprocess() {
     # Take care that its correctly mounted:
-- 
2.20.1

Re: [PATCH v5] meta/classes: generate bill of material from image

[Jan Kiszka]

On 13.08.19 15:40, [ext] Q. Gylstorff wrote:
> From: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> 
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:
> 
> source name| source version | binary package name | binary version
> 
> All rootfs generate the list by default. Currently the f
> following lists are generated:
> - buildchroot-${DISTRO}-${ARCH}.manifest
> - ${IMAGE}-${DISTRO}-${ARCH}.manifest
> - optional: sdkchroot-${DISTRO}-${ARCH}.manifest
> 
> Remove the feature with:
> ROOTFS_FEATURES_remove = "generate-manifest"
> 
> Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
> ---
>   meta/classes/image.bbclass  |  2 +-
>   meta/classes/rootfs.bbclass | 12 +++++++++++-
>   2 files changed, 12 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index ec6bd39..70c46d2 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -58,7 +58,7 @@ image_do_mounts() {
>   }
>   
>   ROOTFSDIR = "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
> +ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
>   ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
>   
>   inherit rootfs
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index c7e0435..59edcde 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -10,7 +10,8 @@ ROOTFS_PACKAGES ?= ""
>   # 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
>   # 'clean-package-cache' - delete package cache from rootfs
>   # 'finalize-rootfs' - delete files needed to chroot into the rootfs
> -ROOTFS_FEATURES ?= ""
> +# 'generate-manifest' - generate a package manifest of the rootfs into ${IMAGE_DEPLOY_DIR}
> +ROOTFS_FEATURES ?= "generate-manifest"
>   
>   ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
>   
> @@ -212,6 +213,15 @@ rootfs_postprocess_finalize() {
>   EOSUDO
>   }
>   
> +ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'rootfs_generate_manifest', '', d)}"
> +rootfs_generate_manifest () {
> +    mkdir -p ${DEPLOY_DIR_IMAGE}
> +    sudo -E chroot '${ROOTFSDIR}' \
> +    dpkg-query \
> +    -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
> +    ${DEPLOY_DIR_IMAGE}/"${PF}".manifest
> +}
> +
>   do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
>   python do_rootfs_postprocess() {
>       # Take care that its correctly mounted:
> 

This "nicely" triggers [1]. And after running bitbake-diffsigs, it's 
also clear why:

$ bitbake-diffsigs tmp/stamps/debian-stretch-armhf/buildchroot-host/1.0-r0.do_rootfs_postprocess.sigdata.8ea37df88464fe5477148b167fd61c80 tmp/stamps/debian-stretch-armhf/buildchroot-host/1.0-r0.do_rootfs_postprocess.sigdata.fae265f1d4686b64bc01ddb05f972db6
basehash changed from dd03388909f6c4c2b905ebaf2539a435 to 2f9b6886ae7f15d5288e7d0ceb493750
Variable MACHINE value changed from 'qemuarm' to 'de0-nano-soc'

You pull an image dependency into the buildchroot by using 
DEPLOY_DIR_IMAGE. And that opens the doors to parallel-build hell. It's 
also semantically incorrect for the buildchroot because those have 
nothing image-specific in them.

The good news: After my patch series, there are also 
DEPLOY_DIR_BUILDCHROOT and DEPLOY_DIR_SDKCHROOT. The bad news: You need 
to find out which one to use.

Jan

[1] https://groups.google.com/d/msgid/isar-users/c93a7dbdbc9448afc0d0b65ec754ac698ed658c6.1566800787.git.jan.kiszka%40siemens.com

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

[PATCH v6] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

All rootfs generate the list by default. Currently the f
following lists are generated:
- buildchroot-${DISTRO}-${ARCH}.manifest
- ${IMAGE}-${DISTRO}-${ARCH}.manifest
- optional: sdkchroot-${DISTRO}-${ARCH}.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---

Changes:
v6:
Add ROOTFS_MANIFEST_DEPLOY_DIR variable to avoid [1].

[1]: https://groups.google.com/d/msgid/isar-users/c93a7dbdbc9448afc0d0b65ec754ac698ed658c6.1566800787.git.jan.kiszka%40siemens.com
v5:
Add logic to rootfs.bbclass instead on recipe
v4:
Add sdk rootfs to manifest
Avoid duplicated code and move gen_accounts_array and gen_manifest_array to
  shell-list-processing-helper
call dpkg-query from $PATH

v3:
Add list of manifest for buildchroot manifest
This list can be exdent to add additional output generators 

v2:
use FEATURE instead of own variable


 meta/classes/image.bbclass                        |  3 ++-
 meta/classes/rootfs.bbclass                       | 11 +++++++++++
 meta/recipes-devtools/buildchroot/buildchroot.inc |  2 +-
 meta/recipes-devtools/sdkchroot/sdkchroot.bb      |  1 +
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 0965f26..4713555 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -61,8 +61,9 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
+ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 
 inherit rootfs
 inherit image-sdk-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 314efd7..405080f 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -5,11 +5,13 @@ ROOTFS_ARCH ?= "${DISTRO_ARCH}"
 ROOTFS_DISTRO ?= "${DISTRO}"
 ROOTFS_PACKAGES ?= ""
 
+
 # Features of the rootfs creation:
 # available features are:
 # 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
 # 'clean-package-cache' - delete package cache from rootfs
 # 'finalize-rootfs' - delete files needed to chroot into the rootfs
+# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
 ROOTFS_FEATURES ?= ""
 
 ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
@@ -222,6 +224,15 @@ rootfs_postprocess_finalize() {
 EOSUDO
 }
 
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'rootfs_generate_manifest', '', d)}"
+rootfs_generate_manifest () {
+    mkdir -p ${ROOTFS_MANIFEST_DEPLOY_DIR}
+    sudo -E chroot --userspec=$(id -u):$(id -g) '${ROOTFSDIR}' \
+    dpkg-query \
+    -f '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' -W > \
+    ${ROOTFS_MANIFEST_DEPLOY_DIR}/"${PF}".manifest
+}
+
 do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
 python do_rootfs_postprocess() {
     # Take care that its correctly mounted:
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
index fc29690..230294b 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.inc
+++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
@@ -19,6 +19,7 @@ BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
 ROOTFSDIR = "${BUILDCHROOT_DIR}"
 ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
 ROOTFS_CLEAN_FILES = ""
+ROOTFS_MANIFEST_DEPLOY_DIR = "${DEPLOY_DIR_BUILDCHROOT}"
 
 BUILDCHROOT_PREINSTALL_COMMON = " \
     make \
@@ -61,7 +62,6 @@ buildchroot_install_files() {
 }
 
 DEPLOY_BUILDCHROOT = "${@d.getVar('BUILDCHROOT_' + d.getVar('BUILDCHROOT_VARIANT').upper() + '_DIR')}"
-
 do_buildchroot_deploy[dirs] = "${DEPLOY_DIR_BUILDCHROOT}-${BUILDCHROOT_VARIANT}"
 do_buildchroot_deploy() {
     ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_BUILDCHROOT}"
diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
index e658122..d2a393b 100644
--- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
+++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
@@ -23,6 +23,7 @@ ROOTFS_DISTRO = "${HOST_DISTRO}"
 ROOTFSDIR = "${S}"
 ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
 ROOTFS_FEATURES += "clean-package-cache"
+ROOTFS_MANIFEST_DEPLOY_DIR = "${DEPLOY_DIR_SDKCHROOT}"
 
 python() {
     if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
-- 
2.20.1

[PATCH v7] meta/classes: generate bill of material from image

[Q. Gylstorff]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

To create products it is necessary to have a list
of used packages for clearance and to security monitoring.
To get a simple list of packages use dpkg-query and generate
a list with the following pattern:

source name| source version | binary package name | binary version

All rootfs generate the list by default. Currently the f
following lists are generated:
- buildchroot-${DISTRO}-${ARCH}.manifest
- ${IMAGE}-${DISTRO}-${ARCH}.manifest
- optional: sdkchroot-${DISTRO}-${ARCH}.manifest

Remove the feature with:
ROOTFS_FEATURES_remove = "generate-manifest"

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
Changes:
v7:
 Whitespaces
v6:
 Add ROOTFS_MANIFEST_DEPLOY_DIR variable to avoid [1].

 [1]: https://groups.google.com/d/msgid/isar-users/c93a7dbdbc9448afc0d0b65ec754ac698ed658c6.1566800787.git.jan.kiszka%40siemens.com
v5:
 Add logic to rootfs.bbclass instead on recipe
v4:
 Add sdk rootfs to manifest
 Avoid duplicated code and move gen_accounts_array and gen_manifest_array to
  shell-list-processing-helper
 call dpkg-query from $PATH

v3:
 Add list of manifest for buildchroot manifest
 This list can be exdent to add additional output generators 

v2:
 use FEATURE instead of own variable

 meta/classes/image.bbclass                        |  3 ++-
 meta/classes/rootfs.bbclass                       | 10 ++++++++++
 meta/recipes-devtools/buildchroot/buildchroot.inc |  2 ++
 meta/recipes-devtools/sdkchroot/sdkchroot.bb      |  3 ++-
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 0965f26..4713555 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -61,8 +61,9 @@ image_do_mounts() {
 }
 
 ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs"
+ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
 ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
+ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
 
 inherit rootfs
 inherit image-sdk-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 314efd7..4dd22b4 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -10,6 +10,7 @@ ROOTFS_PACKAGES ?= ""
 # 'deploy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
 # 'clean-package-cache' - delete package cache from rootfs
 # 'finalize-rootfs' - delete files needed to chroot into the rootfs
+# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
 ROOTFS_FEATURES ?= ""
 
 ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
@@ -222,6 +223,15 @@ rootfs_postprocess_finalize() {
 EOSUDO
 }
 
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-manifest', 'rootfs_generate_manifest', '', d)}"
+rootfs_generate_manifest () {
+    mkdir -p ${ROOTFS_MANIFEST_DEPLOY_DIR}
+    sudo -E chroot --userspec=$(id -u):$(id -g) '${ROOTFSDIR}' \
+        dpkg-query -W -f \
+            '${source:Package}|${source:Version}|${binary:Package}|${Version}\n' > \
+        ${ROOTFS_MANIFEST_DEPLOY_DIR}/"${PF}".manifest
+}
+
 do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
 python do_rootfs_postprocess() {
     # Take care that its correctly mounted:
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.inc b/meta/recipes-devtools/buildchroot/buildchroot.inc
index fc29690..2a48609 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.inc
+++ b/meta/recipes-devtools/buildchroot/buildchroot.inc
@@ -19,6 +19,8 @@ BUILDCHROOT_DIR = "${WORKDIR}/rootfs"
 ROOTFSDIR = "${BUILDCHROOT_DIR}"
 ROOTFS_PACKAGES = "${BUILDCHROOT_PREINSTALL}"
 ROOTFS_CLEAN_FILES = ""
+ROOTFS_MANIFEST_DEPLOY_DIR = "${DEPLOY_DIR_BUILDCHROOT}"
+ROOTFS_FEATURES += "generate-manifest"
 
 BUILDCHROOT_PREINSTALL_COMMON = " \
     make \
diff --git a/meta/recipes-devtools/sdkchroot/sdkchroot.bb b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
index e658122..467e682 100644
--- a/meta/recipes-devtools/sdkchroot/sdkchroot.bb
+++ b/meta/recipes-devtools/sdkchroot/sdkchroot.bb
@@ -22,7 +22,8 @@ ROOTFS_ARCH = "${HOST_ARCH}"
 ROOTFS_DISTRO = "${HOST_DISTRO}"
 ROOTFSDIR = "${S}"
 ROOTFS_PACKAGES = "${SDKCHROOT_PREINSTALL} ${TOOLCHAIN}"
-ROOTFS_FEATURES += "clean-package-cache"
+ROOTFS_FEATURES += "clean-package-cache generate-manifest"
+ROOTFS_MANIFEST_DEPLOY_DIR = "${DEPLOY_DIR_SDKCHROOT}"
 
 python() {
     if d.getVar("HOST_ARCH") not in ['i386', 'amd64']:
-- 
2.20.1

Re: [PATCH v7] meta/classes: generate bill of material from image

[Baurzhan Ismagulov]

On Mon, Sep 23, 2019 at 03:51:54PM +0200, Q. Gylstorff wrote:
> To create products it is necessary to have a list
> of used packages for clearance and to security monitoring.
> To get a simple list of packages use dpkg-query and generate
> a list with the following pattern:

Thanks, applied to next with minor comment updates.

With kind regards,
Baurzhan.