[PATCH] wic-img: handle variables in .wks files

[PATCH] wic-img: handle variables in .wks files

[Cedric Hombourger]

Isar will now generate .wks from user-specified templates with variables
such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their values. Custom
variables may be substituted by adding them to WKSVARS (WKSVAR += FOO)

Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
---
 doc/user_manual.md           |  2 ++
 meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/doc/user_manual.md b/doc/user_manual.md
index ebc31c6..ea3b4bd 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
  $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
 ```
 
+Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables (as well as any other bitbake variables added to WKSVARS).
+
 In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
 https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
 
diff --git a/meta/classes/wic-img.bbclass b/meta/classes/wic-img.bbclass
index 76602d8..16bbc53 100644
--- a/meta/classes/wic-img.bbclass
+++ b/meta/classes/wic-img.bbclass
@@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files before do_wic_image
 do_rootfs_wicenv[vardeps] += "${WICVARS}"
 do_rootfs_wicenv[prefuncs] = 'set_image_size'
 
+WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
+
+python do_rootfs_wksenv () {
+    wksvars = d.getVar('WKSVARS', True)
+    if not wksvars:
+        return
+
+    stdir = d.getVar('STAGING_DIR', True)
+    outdir = os.path.join(stdir, d.getVar('MACHINE', True), 'imgdata')
+    bb.utils.mkdirhier(outdir)
+    basename = d.getVar('IMAGE_BASENAME', True)
+    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as envf:
+        for var in wksvars.split():
+            value = d.getVar(var, True)
+            if value:
+                envf.write('%s="%s" \\\n' % (var, value.strip()))
+        envf.write("envsubst '")
+        for var in wksvars.split():
+            envf.write('$%s ' % var)
+        envf.write("'\n")
+}
+
+addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image
+do_rootfs_wksenv[vardeps] += "${WKSVARS}"
+do_rootfs_wksenv[prefuncs] = 'set_image_size'
+
 WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
 
 do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
@@ -99,8 +125,11 @@ do_wic_image() {
     export BUILDDIR=${BUILDDIR}
     export MTOOLS_SKIP_CHECK=1
 
+    /bin/sh ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
+        <${WKS_FULL_PATH} >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks
+
     sudo -E chroot ${BUILDCHROOT_DIR} \
-        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
+        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
             --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
             -o /tmp/${IMAGE_FULLNAME}.wic/ \
             -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}
-- 
2.11.0

Re: [PATCH] wic-img: handle variables in .wks files

[Jan Kiszka]

On 08.02.19 08:02, Cedric Hombourger wrote:
> Isar will now generate .wks from user-specified templates with variables
> such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their values. Custom
> variables may be substituted by adding them to WKSVARS (WKSVAR += FOO)

Isn't that what WICVARS are for? That's at least what OE is doing. I don't see 
the reason to invent our own interface here.

Jan

> 
> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>   doc/user_manual.md           |  2 ++
>   meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>   2 files changed, 32 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index ebc31c6..ea3b4bd 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
>    $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
>   ```
>   
> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables (as well as any other bitbake variables added to WKSVARS).
> +
>   In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
>   https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
>   
> diff --git a/meta/classes/wic-img.bbclass b/meta/classes/wic-img.bbclass
> index 76602d8..16bbc53 100644
> --- a/meta/classes/wic-img.bbclass
> +++ b/meta/classes/wic-img.bbclass
> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files before do_wic_image
>   do_rootfs_wicenv[vardeps] += "${WICVARS}"
>   do_rootfs_wicenv[prefuncs] = 'set_image_size'
>   
> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
> +
> +python do_rootfs_wksenv () {
> +    wksvars = d.getVar('WKSVARS', True)
> +    if not wksvars:
> +        return
> +
> +    stdir = d.getVar('STAGING_DIR', True)
> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True), 'imgdata')
> +    bb.utils.mkdirhier(outdir)
> +    basename = d.getVar('IMAGE_BASENAME', True)
> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as envf:
> +        for var in wksvars.split():
> +            value = d.getVar(var, True)
> +            if value:
> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
> +        envf.write("envsubst '")
> +        for var in wksvars.split():
> +            envf.write('$%s ' % var)
> +        envf.write("'\n")
> +}
> +
> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image
> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
> +
>   WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>   
>   do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> @@ -99,8 +125,11 @@ do_wic_image() {
>       export BUILDDIR=${BUILDDIR}
>       export MTOOLS_SKIP_CHECK=1
>   
> +    /bin/sh ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
> +        <${WKS_FULL_PATH} >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks
> +
>       sudo -E chroot ${BUILDCHROOT_DIR} \
> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>               --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>               -o /tmp/${IMAGE_FULLNAME}.wic/ \
>               -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

RE: [PATCH] wic-img: handle variables in .wks files

[Hombourger, Cedric]

I will double check but WICVARS are for variables you wish to export to wic plugins
This interface is for variables we would to make available inside .wks files

I realize I did not include any examples in reference .wks files we ship with Isar
I will add some

Cedric

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com] 
Sent: Friday, February 8, 2019 8:42 AM
To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>; isar-users@googlegroups.com
Subject: Re: [PATCH] wic-img: handle variables in .wks files

On 08.02.19 08:02, Cedric Hombourger wrote:
> Isar will now generate .wks from user-specified templates with 
> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their 
> values. Custom variables may be substituted by adding them to WKSVARS 
> (WKSVAR += FOO)

Isn't that what WICVARS are for? That's at least what OE is doing. I don't see the reason to invent our own interface here.

Jan

> 
> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>   doc/user_manual.md           |  2 ++
>   meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>   2 files changed, 32 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md index 
> ebc31c6..ea3b4bd 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
>    $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
>   ```
>   
> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables (as well as any other bitbake variables added to WKSVARS).
> +
>   In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
>   
> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd13547
> 8237cb8f7
>   
> diff --git a/meta/classes/wic-img.bbclass 
> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
> --- a/meta/classes/wic-img.bbclass
> +++ b/meta/classes/wic-img.bbclass
> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files before do_wic_image
>   do_rootfs_wicenv[vardeps] += "${WICVARS}"
>   do_rootfs_wicenv[prefuncs] = 'set_image_size'
>   
> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
> +
> +python do_rootfs_wksenv () {
> +    wksvars = d.getVar('WKSVARS', True)
> +    if not wksvars:
> +        return
> +
> +    stdir = d.getVar('STAGING_DIR', True)
> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True), 'imgdata')
> +    bb.utils.mkdirhier(outdir)
> +    basename = d.getVar('IMAGE_BASENAME', True)
> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as envf:
> +        for var in wksvars.split():
> +            value = d.getVar(var, True)
> +            if value:
> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
> +        envf.write("envsubst '")
> +        for var in wksvars.split():
> +            envf.write('$%s ' % var)
> +        envf.write("'\n")
> +}
> +
> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image 
> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
> +
>   WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>   
>   do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> @@ -99,8 +125,11 @@ do_wic_image() {
>       export BUILDDIR=${BUILDDIR}
>       export MTOOLS_SKIP_CHECK=1
>   
> +    /bin/sh ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
> +        <${WKS_FULL_PATH} 
> + >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks
> +
>       sudo -E chroot ${BUILDCHROOT_DIR} \
> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>               --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>               -o /tmp/${IMAGE_FULLNAME}.wic/ \
>               -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}
> 

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux

Re: [PATCH] wic-img: handle variables in .wks files

[Jan Kiszka]

On 08.02.19 08:44, Hombourger, Cedric wrote:
> I will double check but WICVARS are for variables you wish to export to wic plugins
> This interface is for variables we would to make available inside .wks files

There is no WKSVARS in OE, while you can use variables, see e.g. 
scripts/lib/wic/canned-wks/directdisk.wks.

Jan

> 
> I realize I did not include any examples in reference .wks files we ship with Isar
> I will add some
> 
> Cedric
> 
> -----Original Message-----
> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
> Sent: Friday, February 8, 2019 8:42 AM
> To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>; isar-users@googlegroups.com
> Subject: Re: [PATCH] wic-img: handle variables in .wks files
> 
> On 08.02.19 08:02, Cedric Hombourger wrote:
>> Isar will now generate .wks from user-specified templates with
>> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their
>> values. Custom variables may be substituted by adding them to WKSVARS
>> (WKSVAR += FOO)
> 
> Isn't that what WICVARS are for? That's at least what OE is doing. I don't see the reason to invent our own interface here.
> 
> Jan
> 
>>
>> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
>> ---
>>    doc/user_manual.md           |  2 ++
>>    meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>>    2 files changed, 32 insertions(+), 1 deletion(-)
>>
>> diff --git a/doc/user_manual.md b/doc/user_manual.md index
>> ebc31c6..ea3b4bd 100644
>> --- a/doc/user_manual.md
>> +++ b/doc/user_manual.md
>> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
>>     $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
>>    ```
>>    
>> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables (as well as any other bitbake variables added to WKSVARS).
>> +
>>    In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
>>    
>> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd13547
>> 8237cb8f7
>>    
>> diff --git a/meta/classes/wic-img.bbclass
>> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
>> --- a/meta/classes/wic-img.bbclass
>> +++ b/meta/classes/wic-img.bbclass
>> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files before do_wic_image
>>    do_rootfs_wicenv[vardeps] += "${WICVARS}"
>>    do_rootfs_wicenv[prefuncs] = 'set_image_size'
>>    
>> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
>> +
>> +python do_rootfs_wksenv () {
>> +    wksvars = d.getVar('WKSVARS', True)
>> +    if not wksvars:
>> +        return
>> +
>> +    stdir = d.getVar('STAGING_DIR', True)
>> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True), 'imgdata')
>> +    bb.utils.mkdirhier(outdir)
>> +    basename = d.getVar('IMAGE_BASENAME', True)
>> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as envf:
>> +        for var in wksvars.split():
>> +            value = d.getVar(var, True)
>> +            if value:
>> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
>> +        envf.write("envsubst '")
>> +        for var in wksvars.split():
>> +            envf.write('$%s ' % var)
>> +        envf.write("'\n")
>> +}
>> +
>> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image
>> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
>> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
>> +
>>    WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>>    
>>    do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>> @@ -99,8 +125,11 @@ do_wic_image() {
>>        export BUILDDIR=${BUILDDIR}
>>        export MTOOLS_SKIP_CHECK=1
>>    
>> +    /bin/sh ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
>> +        <${WKS_FULL_PATH}
>> + >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks
>> +
>>        sudo -E chroot ${BUILDCHROOT_DIR} \
>> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
>> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>>                --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>>                -o /tmp/${IMAGE_FULLNAME}.wic/ \
>>                -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}
>>
> 
> --
> Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] wic-img: handle variables in .wks files

[Jan Kiszka]

On 08.02.19 08:48, [ext] Jan Kiszka wrote:
> On 08.02.19 08:44, Hombourger, Cedric wrote:
>> I will double check but WICVARS are for variables you wish to export to wic 
>> plugins
>> This interface is for variables we would to make available inside .wks files
> 
> There is no WKSVARS in OE, while you can use variables, see e.g. 
> scripts/lib/wic/canned-wks/directdisk.wks.
> 

Sorry, I meant scripts/lib/wic/canned-wks/efi-bootdisk.wks.in.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

RE: [PATCH] wic-img: handle variables in .wks files

[Hombourger, Cedric]

Thanks for the pointer. Will check how that's implemented there and if it works equally well on Isar
(I was obviously not aware that OE was now having this)

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com] 
Sent: Friday, February 8, 2019 8:53 AM
To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>; isar-users@googlegroups.com
Subject: Re: [PATCH] wic-img: handle variables in .wks files

On 08.02.19 08:48, [ext] Jan Kiszka wrote:
> On 08.02.19 08:44, Hombourger, Cedric wrote:
>> I will double check but WICVARS are for variables you wish to export 
>> to wic plugins This interface is for variables we would to make 
>> available inside .wks files
> 
> There is no WKSVARS in OE, while you can use variables, see e.g. 
> scripts/lib/wic/canned-wks/directdisk.wks.
> 

Sorry, I meant scripts/lib/wic/canned-wks/efi-bootdisk.wks.in.

Jan

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux

RE: [PATCH] wic-img: handle variables in .wks files

[Hombourger, Cedric]

Hi Jan,

JFYI - it does not seem to work out of the box with Isar (despite the variables that I wanted expanded being present in the .env file we are passing to wic via --vars)
The magic is done by the image_types_wic class, see below for specifics

Will import this code in Isar so we are better aligned and post a v2

Cedric

python do_write_wks_template () {
    """Write out expanded template contents to WKS_FULL_PATH."""
    import re

    template_body = d.getVar('_WKS_TEMPLATE')

    # Remove any remnant variable references left behind by the expansion
    # due to undefined variables
    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
    while True:
        new_body = re.sub(expand_var_regexp, '', template_body)
        if new_body == template_body:
            break
        else:
            template_body = new_body

    wks_file = d.getVar('WKS_FULL_PATH')
    with open(wks_file, 'w') as f:
        f.write(template_body)
}

python () {
    if d.getVar('USING_WIC'):
        wks_file_u = d.getVar('WKS_FULL_PATH', False)
        wks_file = d.expand(wks_file_u)
        base, ext = os.path.splitext(wks_file)
        if ext == '.in' and os.path.exists(wks_file):
            wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
            d.setVar('WKS_FULL_PATH', wks_out_file)
            d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
            d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
           
            <snip> magic continues</snip>

            bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d)

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com] 
Sent: Friday, February 8, 2019 8:53 AM
To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>; isar-users@googlegroups.com
Subject: Re: [PATCH] wic-img: handle variables in .wks files

On 08.02.19 08:48, [ext] Jan Kiszka wrote:
> On 08.02.19 08:44, Hombourger, Cedric wrote:
>> I will double check but WICVARS are for variables you wish to export 
>> to wic plugins This interface is for variables we would to make 
>> available inside .wks files
> 
> There is no WKSVARS in OE, while you can use variables, see e.g. 
> scripts/lib/wic/canned-wks/directdisk.wks.
> 

Sorry, I meant scripts/lib/wic/canned-wks/efi-bootdisk.wks.in.

Jan

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux

RE: [PATCH] wic-img: handle variables in .wks files

[Hombourger, Cedric]

Code imported and working in our environment. I am now going to run a full ci-build test to make sure no (tested) configurations are broken and submit a v2

-----Original Message-----
From: Hombourger, Cedric 
Sent: Friday, February 8, 2019 12:32 PM
To: 'Jan Kiszka' <jan.kiszka@siemens.com>; isar-users@googlegroups.com
Subject: RE: [PATCH] wic-img: handle variables in .wks files

Hi Jan,

JFYI - it does not seem to work out of the box with Isar (despite the variables that I wanted expanded being present in the .env file we are passing to wic via --vars) The magic is done by the image_types_wic class, see below for specifics

Will import this code in Isar so we are better aligned and post a v2

Cedric

python do_write_wks_template () {
    """Write out expanded template contents to WKS_FULL_PATH."""
    import re

    template_body = d.getVar('_WKS_TEMPLATE')

    # Remove any remnant variable references left behind by the expansion
    # due to undefined variables
    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
    while True:
        new_body = re.sub(expand_var_regexp, '', template_body)
        if new_body == template_body:
            break
        else:
            template_body = new_body

    wks_file = d.getVar('WKS_FULL_PATH')
    with open(wks_file, 'w') as f:
        f.write(template_body)
}

python () {
    if d.getVar('USING_WIC'):
        wks_file_u = d.getVar('WKS_FULL_PATH', False)
        wks_file = d.expand(wks_file_u)
        base, ext = os.path.splitext(wks_file)
        if ext == '.in' and os.path.exists(wks_file):
            wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
            d.setVar('WKS_FULL_PATH', wks_out_file)
            d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
            d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
           
            <snip> magic continues</snip>

            bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d)

-----Original Message-----
From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
Sent: Friday, February 8, 2019 8:53 AM
To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>; isar-users@googlegroups.com
Subject: Re: [PATCH] wic-img: handle variables in .wks files

On 08.02.19 08:48, [ext] Jan Kiszka wrote:
> On 08.02.19 08:44, Hombourger, Cedric wrote:
>> I will double check but WICVARS are for variables you wish to export 
>> to wic plugins This interface is for variables we would to make 
>> available inside .wks files
> 
> There is no WKSVARS in OE, while you can use variables, see e.g. 
> scripts/lib/wic/canned-wks/directdisk.wks.
> 

Sorry, I meant scripts/lib/wic/canned-wks/efi-bootdisk.wks.in.

Jan

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux

Re: [PATCH] wic-img: handle variables in .wks files

[Henning Schild]

Not sure i get all the context right ... But if you are introducing a
template mechanism to generate .wks files, have a look at the patches
from Claudius that are currently pending.
He introduces a generic templating mechanism which may be useful for
generating .wks files as well.

[PATCH v2 1/1] meta: added do_transform_template task as templating
system and switch

You could give that a try and maybe still vote for or against that
mechanism. At the moment i would say it is likely to go in, and
consequently other/future templating mechanism will we be questioned.

Henning

Am Fri, 8 Feb 2019 08:02:47 +0100
schrieb Cedric Hombourger <Cedric_Hombourger@mentor.com>:

> Isar will now generate .wks from user-specified templates with
> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their
> values. Custom variables may be substituted by adding them to WKSVARS
> (WKSVAR += FOO)
> 
> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>  doc/user_manual.md           |  2 ++
>  meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>  2 files changed, 32 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index ebc31c6..ea3b4bd 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set
> IMAGE_TYPE to 'wic-img'. Behind th $ bitbake
> multiconfig:qemuamd64-stretch:isar-image-base ```
>  
> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables
> (as well as any other bitbake variables added to WKSVARS). +
>  In order to run the EFI images with `qemu`, an EFI firmware is
> required and available at the following address:
> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7 
> diff --git a/meta/classes/wic-img.bbclass
> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
> --- a/meta/classes/wic-img.bbclass
> +++ b/meta/classes/wic-img.bbclass
> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files
> before do_wic_image do_rootfs_wicenv[vardeps] += "${WICVARS}"
>  do_rootfs_wicenv[prefuncs] = 'set_image_size'
>  
> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
> +
> +python do_rootfs_wksenv () {
> +    wksvars = d.getVar('WKSVARS', True)
> +    if not wksvars:
> +        return
> +
> +    stdir = d.getVar('STAGING_DIR', True)
> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True),
> 'imgdata')
> +    bb.utils.mkdirhier(outdir)
> +    basename = d.getVar('IMAGE_BASENAME', True)
> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as
> envf:
> +        for var in wksvars.split():
> +            value = d.getVar(var, True)
> +            if value:
> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
> +        envf.write("envsubst '")
> +        for var in wksvars.split():
> +            envf.write('$%s ' % var)
> +        envf.write("'\n")
> +}
> +
> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image
> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
> +
>  WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>  
>  do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> @@ -99,8 +125,11 @@ do_wic_image() {
>      export BUILDDIR=${BUILDDIR}
>      export MTOOLS_SKIP_CHECK=1
>  
> +    /bin/sh
> ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
> +        <${WKS_FULL_PATH}
> >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks +
>      sudo -E chroot ${BUILDCHROOT_DIR} \
> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>              --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>              -o /tmp/${IMAGE_FULLNAME}.wic/ \
>              -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}

RE: [PATCH] wic-img: handle variables in .wks files

[Hombourger, Cedric]

Hi Henning,

Will surely take a look but we will have to decide whether we'd want our wic-img class to be closely aligned with oe or accept this divergence
(note that I am not suggesting any options right now as I should take a look at the framework proposed by Claudius)

Cedric

-----Original Message-----
From: Henning Schild [mailto:henning.schild@siemens.com] 
Sent: Friday, February 8, 2019 3:29 PM
To: Hombourger, Cedric <Cedric_Hombourger@mentor.com>
Cc: isar-users@googlegroups.com
Subject: Re: [PATCH] wic-img: handle variables in .wks files

Not sure i get all the context right ... But if you are introducing a template mechanism to generate .wks files, have a look at the patches from Claudius that are currently pending.
He introduces a generic templating mechanism which may be useful for generating .wks files as well.

[PATCH v2 1/1] meta: added do_transform_template task as templating system and switch

You could give that a try and maybe still vote for or against that mechanism. At the moment i would say it is likely to go in, and consequently other/future templating mechanism will we be questioned.

Henning

Am Fri, 8 Feb 2019 08:02:47 +0100
schrieb Cedric Hombourger <Cedric_Hombourger@mentor.com>:

> Isar will now generate .wks from user-specified templates with 
> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their 
> values. Custom variables may be substituted by adding them to WKSVARS 
> (WKSVAR += FOO)
> 
> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>  doc/user_manual.md           |  2 ++
>  meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>  2 files changed, 32 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md index 
> ebc31c6..ea3b4bd 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set 
> IMAGE_TYPE to 'wic-img'. Behind th $ bitbake 
> multiconfig:qemuamd64-stretch:isar-image-base ```
>  
> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables
> (as well as any other bitbake variables added to WKSVARS). +  In order 
> to run the EFI images with `qemu`, an EFI firmware is required and 
> available at the following address:
> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd13547
> 8237cb8f7 diff --git a/meta/classes/wic-img.bbclass 
> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
> --- a/meta/classes/wic-img.bbclass
> +++ b/meta/classes/wic-img.bbclass
> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files 
> before do_wic_image do_rootfs_wicenv[vardeps] += "${WICVARS}"
>  do_rootfs_wicenv[prefuncs] = 'set_image_size'
>  
> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
> +
> +python do_rootfs_wksenv () {
> +    wksvars = d.getVar('WKSVARS', True)
> +    if not wksvars:
> +        return
> +
> +    stdir = d.getVar('STAGING_DIR', True)
> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True),
> 'imgdata')
> +    bb.utils.mkdirhier(outdir)
> +    basename = d.getVar('IMAGE_BASENAME', True)
> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as
> envf:
> +        for var in wksvars.split():
> +            value = d.getVar(var, True)
> +            if value:
> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
> +        envf.write("envsubst '")
> +        for var in wksvars.split():
> +            envf.write('$%s ' % var)
> +        envf.write("'\n")
> +}
> +
> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image 
> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
> +
>  WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>  
>  do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> @@ -99,8 +125,11 @@ do_wic_image() {
>      export BUILDDIR=${BUILDDIR}
>      export MTOOLS_SKIP_CHECK=1
>  
> +    /bin/sh
> ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
> +        <${WKS_FULL_PATH}
> >${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks +
>      sudo -E chroot ${BUILDCHROOT_DIR} \
> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>              --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>              -o /tmp/${IMAGE_FULLNAME}.wic/ \
>              -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}

Re: [PATCH] wic-img: handle variables in .wks files

[Jan Kiszka]

On 08.02.19 15:28, [ext] Henning Schild wrote:
> Not sure i get all the context right ... But if you are introducing a
> template mechanism to generate .wks files, have a look at the patches
> from Claudius that are currently pending.
> He introduces a generic templating mechanism which may be useful for
> generating .wks files as well.

That is a different mechanism, not intended for patching recipe files (which wks 
files are). We align with OE here as far as I can see.

Jan

> 
> [PATCH v2 1/1] meta: added do_transform_template task as templating
> system and switch
> 
> You could give that a try and maybe still vote for or against that
> mechanism. At the moment i would say it is likely to go in, and
> consequently other/future templating mechanism will we be questioned.
> 
> Henning
> 
> Am Fri, 8 Feb 2019 08:02:47 +0100
> schrieb Cedric Hombourger <Cedric_Hombourger@mentor.com>:
> 
>> Isar will now generate .wks from user-specified templates with
>> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their
>> values. Custom variables may be substituted by adding them to WKSVARS
>> (WKSVAR += FOO)
>>
>> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
>> ---
>>   doc/user_manual.md           |  2 ++
>>   meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
>>   2 files changed, 32 insertions(+), 1 deletion(-)
>>
>> diff --git a/doc/user_manual.md b/doc/user_manual.md
>> index ebc31c6..ea3b4bd 100644
>> --- a/doc/user_manual.md
>> +++ b/doc/user_manual.md
>> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set
>> IMAGE_TYPE to 'wic-img'. Behind th $ bitbake
>> multiconfig:qemuamd64-stretch:isar-image-base ```
>>   
>> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE variables
>> (as well as any other bitbake variables added to WKSVARS). +
>>   In order to run the EFI images with `qemu`, an EFI firmware is
>> required and available at the following address:
>> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
>> diff --git a/meta/classes/wic-img.bbclass
>> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
>> --- a/meta/classes/wic-img.bbclass
>> +++ b/meta/classes/wic-img.bbclass
>> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after do_copy_boot_files
>> before do_wic_image do_rootfs_wicenv[vardeps] += "${WICVARS}"
>>   do_rootfs_wicenv[prefuncs] = 'set_image_size'
>>   
>> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
>> +
>> +python do_rootfs_wksenv () {
>> +    wksvars = d.getVar('WKSVARS', True)
>> +    if not wksvars:
>> +        return
>> +
>> +    stdir = d.getVar('STAGING_DIR', True)
>> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True),
>> 'imgdata')
>> +    bb.utils.mkdirhier(outdir)
>> +    basename = d.getVar('IMAGE_BASENAME', True)
>> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as
>> envf:
>> +        for var in wksvars.split():
>> +            value = d.getVar(var, True)
>> +            if value:
>> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
>> +        envf.write("envsubst '")
>> +        for var in wksvars.split():
>> +            envf.write('$%s ' % var)
>> +        envf.write("'\n")
>> +}
>> +
>> +addtask do_rootfs_wksenv after do_rootfs_wicenv before do_wic_image
>> +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
>> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
>> +
>>   WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
>>   
>>   do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>> @@ -99,8 +125,11 @@ do_wic_image() {
>>       export BUILDDIR=${BUILDDIR}
>>       export MTOOLS_SKIP_CHECK=1
>>   
>> +    /bin/sh
>> ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
>> +        <${WKS_FULL_PATH}
>>> ${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks +
>>       sudo -E chroot ${BUILDCHROOT_DIR} \
>> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
>> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks \
>>               --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
>>               -o /tmp/${IMAGE_FULLNAME}.wic/ \
>>               -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}
> 

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH] wic-img: handle variables in .wks files

[Henning Schild]

Am Fri, 8 Feb 2019 15:31:48 +0100
schrieb Jan Kiszka <jan.kiszka@siemens.com>:

> On 08.02.19 15:28, [ext] Henning Schild wrote:
> > Not sure i get all the context right ... But if you are introducing
> > a template mechanism to generate .wks files, have a look at the
> > patches from Claudius that are currently pending.
> > He introduces a generic templating mechanism which may be useful for
> > generating .wks files as well.  
> 
> That is a different mechanism, not intended for patching recipe files
> (which wks files are). We align with OE here as far as I can see.

I would not agree. wks-Files are configuration for a tool we call in a
recipe, just like debian/ is for dpkg-buildpackage.

I guess the following should work with Claudius patches
SRC_URI+=file:///my.wks.tmpl and WKS_FILE=WORKDIR/my.wks

WICVARS are another way of passing configuration to wic, here it is
about all the bitbake vars that your plugins want to access, not about
vars that you might want to substitute in a .wks.tmpl before even
calling wic. Afaik wks files can not contain variables anymore.

But i guess we should all see the examples to understand where the code
is coming from and what it is for.

Henning

> Jan
> 
> > 
> > [PATCH v2 1/1] meta: added do_transform_template task as templating
> > system and switch
> > 
> > You could give that a try and maybe still vote for or against that
> > mechanism. At the moment i would say it is likely to go in, and
> > consequently other/future templating mechanism will we be
> > questioned.
> > 
> > Henning
> > 
> > Am Fri, 8 Feb 2019 08:02:47 +0100
> > schrieb Cedric Hombourger <Cedric_Hombourger@mentor.com>:
> >   
> >> Isar will now generate .wks from user-specified templates with
> >> variables such as $ROOTFS_TYPE or $ROOTFS_SIZE replaced with their
> >> values. Custom variables may be substituted by adding them to
> >> WKSVARS (WKSVAR += FOO)
> >>
> >> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> >> ---
> >>   doc/user_manual.md           |  2 ++
> >>   meta/classes/wic-img.bbclass | 31 ++++++++++++++++++++++++++++++-
> >>   2 files changed, 32 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/doc/user_manual.md b/doc/user_manual.md
> >> index ebc31c6..ea3b4bd 100644
> >> --- a/doc/user_manual.md
> >> +++ b/doc/user_manual.md
> >> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set
> >> IMAGE_TYPE to 'wic-img'. Behind th $ bitbake
> >> multiconfig:qemuamd64-stretch:isar-image-base ```
> >>   
> >> +Note: `.wks` files may use the ROOTFS_SIZE and ROOTFS_TYPE
> >> variables (as well as any other bitbake variables added to
> >> WKSVARS). + In order to run the EFI images with `qemu`, an EFI
> >> firmware is required and available at the following address:
> >> https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
> >> diff --git a/meta/classes/wic-img.bbclass
> >> b/meta/classes/wic-img.bbclass index 76602d8..16bbc53 100644
> >> --- a/meta/classes/wic-img.bbclass
> >> +++ b/meta/classes/wic-img.bbclass
> >> @@ -81,6 +81,32 @@ addtask do_rootfs_wicenv after
> >> do_copy_boot_files before do_wic_image do_rootfs_wicenv[vardeps]
> >> += "${WICVARS}" do_rootfs_wicenv[prefuncs] = 'set_image_size'
> >>   
> >> +WKSVARS += "ROOTFS_SIZE ROOTFS_TYPE"
> >> +
> >> +python do_rootfs_wksenv () {
> >> +    wksvars = d.getVar('WKSVARS', True)
> >> +    if not wksvars:
> >> +        return
> >> +
> >> +    stdir = d.getVar('STAGING_DIR', True)
> >> +    outdir = os.path.join(stdir, d.getVar('MACHINE', True),
> >> 'imgdata')
> >> +    bb.utils.mkdirhier(outdir)
> >> +    basename = d.getVar('IMAGE_BASENAME', True)
> >> +    with open(os.path.join(outdir, basename) + '-wks.sh', 'w') as
> >> envf:
> >> +        for var in wksvars.split():
> >> +            value = d.getVar(var, True)
> >> +            if value:
> >> +                envf.write('%s="%s" \\\n' % (var, value.strip()))
> >> +        envf.write("envsubst '")
> >> +        for var in wksvars.split():
> >> +            envf.write('$%s ' % var)
> >> +        envf.write("'\n")
> >> +}
> >> +
> >> +addtask do_rootfs_wksenv after do_rootfs_wicenv before
> >> do_wic_image +do_rootfs_wksenv[vardeps] += "${WKSVARS}"
> >> +do_rootfs_wksenv[prefuncs] = 'set_image_size'
> >> +
> >>   WIC_IMAGE_FILE ="${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.img"
> >>   
> >>   do_build[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> >> @@ -99,8 +125,11 @@ do_wic_image() {
> >>       export BUILDDIR=${BUILDDIR}
> >>       export MTOOLS_SKIP_CHECK=1
> >>   
> >> +    /bin/sh
> >> ${STAGING_DIR}/${MACHINE}/imgdata/${IMAGE_BASENAME}-wks.sh \
> >> +        <${WKS_FULL_PATH}  
> >>> ${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wks +  
> >>       sudo -E chroot ${BUILDCHROOT_DIR} \
> >> -        ${ISARROOT}/scripts/wic create ${WKS_FULL_PATH} \
> >> +        ${ISARROOT}/scripts/wic create /tmp/${IMAGE_FULLNAME}.wks
> >> \ --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \
> >>               -o /tmp/${IMAGE_FULLNAME}.wic/ \
> >>               -e ${IMAGE_BASENAME} ${WIC_CREATE_EXTRA_ARGS}  
> >   
> 

[PATCH v2] wic-img: import support for .wks.in files from oe-core

[Cedric Hombourger]

OpenEmbedded supports use of variables in wks files if they are
suffixed with .wks.in (instead of .wks). Use of this suffix adds
a task to the build process to expand variables to their values
when known or to the empty string when they are not. The result
of the evaluation is placed in a regular .wks file placed in the
build folder.

The .wks file for the de0-nano-soc machine was modified to show
use of this feature and get it regularly tested as part of our
regular Isar builds.

Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
---
 doc/user_manual.md                                 |  2 +
 meta-isar/conf/machine/de0-nano-soc.conf           |  2 +-
 .../{de0-nano-soc.wks => de0-nano-soc.wks.in}      |  2 +-
 meta/classes/wic-img.bbclass                       | 53 +++++++++++++++++++++-
 4 files changed, 55 insertions(+), 4 deletions(-)
 rename meta-isar/scripts/lib/wic/canned-wks/{de0-nano-soc.wks => de0-nano-soc.wks.in} (63%)

diff --git a/doc/user_manual.md b/doc/user_manual.md
index ebc31c6..252d2ef 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
  $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
 ```
 
+Variables may be used in `.wks.in` files; Isar will expand them and generate a regular `.wks` file before generating the disk image using `wic`.
+
 In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
 https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
 
diff --git a/meta-isar/conf/machine/de0-nano-soc.conf b/meta-isar/conf/machine/de0-nano-soc.conf
index 63b2cfb..3a2c009 100644
--- a/meta-isar/conf/machine/de0-nano-soc.conf
+++ b/meta-isar/conf/machine/de0-nano-soc.conf
@@ -11,7 +11,7 @@ U_BOOT_CONFIG_de0-nano-soc = "socfpga_de0_nano_soc_defconfig"
 U_BOOT_BIN_de0-nano-soc = "u-boot-with-spl.sfp"
 
 IMAGE_TYPE ?= "wic-img"
-WKS_FILE ?= "de0-nano-soc"
+WKS_FILE ?= "de0-nano-soc.wks.in"
 IMAGER_INSTALL += "u-boot-de0-nano-soc"
 IMAGER_BUILD_DEPS += "u-boot-de0-nano-soc"
 
diff --git a/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks b/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
similarity index 63%
rename from meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks
rename to meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
index 184474d..05ce61b 100644
--- a/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks
+++ b/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-part --source rawcopy --sourceparams "file=/usr/lib/u-boot/de0-nano-soc/u-boot-with-spl.sfp" --system-id 0xa2 --align 1
+part --source rawcopy --sourceparams "file=/usr/lib/u-boot/${MACHINE}/${U_BOOT_BIN}" --system-id 0xa2 --align 1
 
 part / --source rootfs-u-boot --ondisk mmcblk0 --fstype ext4 --label platform --align 1024 --active
 
diff --git a/meta/classes/wic-img.bbclass b/meta/classes/wic-img.bbclass
index 76602d8..c9d90a9 100644
--- a/meta/classes/wic-img.bbclass
+++ b/meta/classes/wic-img.bbclass
@@ -4,13 +4,36 @@
 # this class is heavily inspired by OEs ./meta/classes/image_types_wic.bbclass
 #
 
+WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}')}"
+
+python do_write_wks_template () {
+    """Write out expanded template contents to WKS_FULL_PATH."""
+    import re
+
+    template_body = d.getVar('_WKS_TEMPLATE')
+
+    # Remove any remnant variable references left behind by the expansion
+    # due to undefined variables
+    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
+    while True:
+        new_body = re.sub(expand_var_regexp, '', template_body)
+        if new_body == template_body:
+            break
+        else:
+            template_body = new_body
+
+    wks_file = d.getVar('WKS_FULL_PATH')
+    with open(wks_file, 'w') as f:
+        f.write(template_body)
+}
+
 python () {
     wks_full_path = None
 
     wks_file = d.getVar('WKS_FILE', True)
     if not wks_file:
         bb.fatal("WKS_FILE must be set")
-    if not wks_file.endswith('.wks'):
+    if not wks_file.endswith('.wks') and not wks_file.endswith('.wks.in'):
         wks_file += '.wks'
 
     if os.path.isabs(wks_file):
@@ -28,6 +51,32 @@ python () {
         bb.fatal("WKS_FILE '%s' not found" % wks_file)
 
     d.setVar('WKS_FULL_PATH', wks_full_path)
+
+    wks_file_u = wks_full_path
+    wks_file = wks_full_path
+    base, ext = os.path.splitext(wks_file)
+    if ext == '.in' and os.path.exists(wks_file):
+        wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
+        d.setVar('WKS_FULL_PATH', wks_out_file)
+        d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
+        d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
+
+        # We need to re-parse each time the file changes, and bitbake
+        # needs to be told about that explicitly.
+        bb.parse.mark_dependency(d, wks_file)
+
+        try:
+            with open(wks_file, 'r') as f:
+                body = f.read()
+        except (IOError, OSError) as exc:
+            pass
+        else:
+            # Previously, I used expandWithRefs to get the dependency list
+            # and add it to WICVARS, but there's no point re-parsing the
+            # file in process_wks_template as well, so just put it in
+            # a variable and let the metadata deal with the deps.
+            d.setVar('_WKS_TEMPLATE', body)
+            bb.build.addtask('do_write_wks_template', 'do_wic_image', None, d)
 }
 
 inherit buildchroot
@@ -108,7 +157,7 @@ do_wic_image() {
     cp -f $(ls -t -1 ${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wic/*.direct | head -1) ${WIC_IMAGE_FILE}
 }
 
-do_wic_image[file-checksums] += "${WKS_FULL_PATH}:True"
+do_wic_image[file-checksums] += "${WKS_FILE_CHECKSUM}"
 do_wic_image[depends] = "buildchroot-target:do_build"
 
 addtask wic_image before do_build after do_install_imager_deps
-- 
2.11.0

Re: [PATCH v2] wic-img: import support for .wks.in files from oe-core

[Maxim Yu. Osipov]

On 2/8/19 8:13 PM, Cedric Hombourger wrote:
> OpenEmbedded supports use of variables in wks files if they are
> suffixed with .wks.in (instead of .wks). Use of this suffix adds
> a task to the build process to expand variables to their values
> when known or to the empty string when they are not. The result
> of the evaluation is placed in a regular .wks file placed in the
> build folder.
> 
> The .wks file for the de0-nano-soc machine was modified to show
> use of this feature and get it regularly tested as part of our
> regular Isar builds.

Applied to the 'next',

Thanks,
Maxim.

> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>   doc/user_manual.md                                 |  2 +
>   meta-isar/conf/machine/de0-nano-soc.conf           |  2 +-
>   .../{de0-nano-soc.wks => de0-nano-soc.wks.in}      |  2 +-
>   meta/classes/wic-img.bbclass                       | 53 +++++++++++++++++++++-
>   4 files changed, 55 insertions(+), 4 deletions(-)
>   rename meta-isar/scripts/lib/wic/canned-wks/{de0-nano-soc.wks => de0-nano-soc.wks.in} (63%)
> 
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index ebc31c6..252d2ef 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -197,6 +197,8 @@ A bootable disk image is generated if you set IMAGE_TYPE to 'wic-img'. Behind th
>    $ bitbake multiconfig:qemuamd64-stretch:isar-image-base
>   ```
>   
> +Variables may be used in `.wks.in` files; Isar will expand them and generate a regular `.wks` file before generating the disk image using `wic`.
> +
>   In order to run the EFI images with `qemu`, an EFI firmware is required and available at the following address:
>   https://github.com/tianocore/edk2/tree/3858b4a1ff09d3243fea8d07bd135478237cb8f7
>   
> diff --git a/meta-isar/conf/machine/de0-nano-soc.conf b/meta-isar/conf/machine/de0-nano-soc.conf
> index 63b2cfb..3a2c009 100644
> --- a/meta-isar/conf/machine/de0-nano-soc.conf
> +++ b/meta-isar/conf/machine/de0-nano-soc.conf
> @@ -11,7 +11,7 @@ U_BOOT_CONFIG_de0-nano-soc = "socfpga_de0_nano_soc_defconfig"
>   U_BOOT_BIN_de0-nano-soc = "u-boot-with-spl.sfp"
>   
>   IMAGE_TYPE ?= "wic-img"
> -WKS_FILE ?= "de0-nano-soc"
> +WKS_FILE ?= "de0-nano-soc.wks.in"
>   IMAGER_INSTALL += "u-boot-de0-nano-soc"
>   IMAGER_BUILD_DEPS += "u-boot-de0-nano-soc"
>   
> diff --git a/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks b/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
> similarity index 63%
> rename from meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks
> rename to meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
> index 184474d..05ce61b 100644
> --- a/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks
> +++ b/meta-isar/scripts/lib/wic/canned-wks/de0-nano-soc.wks.in
> @@ -3,7 +3,7 @@
>   #
>   # SPDX-License-Identifier: MIT
>   
> -part --source rawcopy --sourceparams "file=/usr/lib/u-boot/de0-nano-soc/u-boot-with-spl.sfp" --system-id 0xa2 --align 1
> +part --source rawcopy --sourceparams "file=/usr/lib/u-boot/${MACHINE}/${U_BOOT_BIN}" --system-id 0xa2 --align 1
>   
>   part / --source rootfs-u-boot --ondisk mmcblk0 --fstype ext4 --label platform --align 1024 --active
>   
> diff --git a/meta/classes/wic-img.bbclass b/meta/classes/wic-img.bbclass
> index 76602d8..c9d90a9 100644
> --- a/meta/classes/wic-img.bbclass
> +++ b/meta/classes/wic-img.bbclass
> @@ -4,13 +4,36 @@
>   # this class is heavily inspired by OEs ./meta/classes/image_types_wic.bbclass
>   #
>   
> +WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}')}"
> +
> +python do_write_wks_template () {
> +    """Write out expanded template contents to WKS_FULL_PATH."""
> +    import re
> +
> +    template_body = d.getVar('_WKS_TEMPLATE')
> +
> +    # Remove any remnant variable references left behind by the expansion
> +    # due to undefined variables
> +    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
> +    while True:
> +        new_body = re.sub(expand_var_regexp, '', template_body)
> +        if new_body == template_body:
> +            break
> +        else:
> +            template_body = new_body
> +
> +    wks_file = d.getVar('WKS_FULL_PATH')
> +    with open(wks_file, 'w') as f:
> +        f.write(template_body)
> +}
> +
>   python () {
>       wks_full_path = None
>   
>       wks_file = d.getVar('WKS_FILE', True)
>       if not wks_file:
>           bb.fatal("WKS_FILE must be set")
> -    if not wks_file.endswith('.wks'):
> +    if not wks_file.endswith('.wks') and not wks_file.endswith('.wks.in'):
>           wks_file += '.wks'
>   
>       if os.path.isabs(wks_file):
> @@ -28,6 +51,32 @@ python () {
>           bb.fatal("WKS_FILE '%s' not found" % wks_file)
>   
>       d.setVar('WKS_FULL_PATH', wks_full_path)
> +
> +    wks_file_u = wks_full_path
> +    wks_file = wks_full_path
> +    base, ext = os.path.splitext(wks_file)
> +    if ext == '.in' and os.path.exists(wks_file):
> +        wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
> +        d.setVar('WKS_FULL_PATH', wks_out_file)
> +        d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
> +        d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
> +
> +        # We need to re-parse each time the file changes, and bitbake
> +        # needs to be told about that explicitly.
> +        bb.parse.mark_dependency(d, wks_file)
> +
> +        try:
> +            with open(wks_file, 'r') as f:
> +                body = f.read()
> +        except (IOError, OSError) as exc:
> +            pass
> +        else:
> +            # Previously, I used expandWithRefs to get the dependency list
> +            # and add it to WICVARS, but there's no point re-parsing the
> +            # file in process_wks_template as well, so just put it in
> +            # a variable and let the metadata deal with the deps.
> +            d.setVar('_WKS_TEMPLATE', body)
> +            bb.build.addtask('do_write_wks_template', 'do_wic_image', None, d)
>   }
>   
>   inherit buildchroot
> @@ -108,7 +157,7 @@ do_wic_image() {
>       cp -f $(ls -t -1 ${BUILDCHROOT_DIR}/tmp/${IMAGE_FULLNAME}.wic/*.direct | head -1) ${WIC_IMAGE_FILE}
>   }
>   
> -do_wic_image[file-checksums] += "${WKS_FULL_PATH}:True"
> +do_wic_image[file-checksums] += "${WKS_FILE_CHECKSUM}"
>   do_wic_image[depends] = "buildchroot-target:do_build"
>   
>   addtask wic_image before do_build after do_install_imager_deps
> 


-- 
Maxim Osipov
ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn
Germany
+49 (151) 6517 6917
mosipov@ilbers.de
http://ilbers.de/
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov