[PATCH v7 1/2] meta/classes: Generate ova image for vmware or virtualbox

["Q. Gylstorff" <Quirin.Gylstorff@siemens.com>]

From: Quirin Gylstorff <quirin.gylstorff@siemens.com>

This allows to generate a ova file for virtualbox or vmware. The
images differ in the setting of the variable `VMDK_SUBFORMAT`.
- `streamOptimized` is used for Vmware Workstation
- `monolithicSparse` is used for Virtualbox

This is necessary as virtualbox throws an import error for a `streamOptimized`
version. The ova for Virtualbox is also bigger due the disk format.

The default machine settings are:
- 4 CPU Cores with 8GB RAM

Signed-off-by: Quirin Gylstorff <quirin.gylstorff@siemens.com>
---
 doc/user_manual.md                            |   1 +
 meta-isar/conf/machine/virtualbox.conf        |  16 ++
 meta-isar/conf/machine/vmware.conf            |  16 ++
 .../multiconfig/virtualbox-ova-buster.conf    |   8 +
 meta/classes/virtual-machine-image.bbclass    |  92 +++++++++++
 .../files/virtual-machine-template.ovf.tmpl   | 155 ++++++++++++++++++
 .../virtual-machine-template_0.1.bb           |  17 ++
 scripts/ci_build.sh                           |   3 +-
 8 files changed, 307 insertions(+), 1 deletion(-)
 create mode 100644 meta-isar/conf/machine/virtualbox.conf
 create mode 100644 meta-isar/conf/machine/vmware.conf
 create mode 100644 meta-isar/conf/multiconfig/virtualbox-ova-buster.conf
 create mode 100644 meta/classes/virtual-machine-image.bbclass
 create mode 100644 meta/recipes-devtools/virtual-machine-template/files/virtual-machine-template.ovf.tmpl
 create mode 100644 meta/recipes-devtools/virtual-machine-template/virtual-machine-template_0.1.bb

diff --git a/doc/user_manual.md b/doc/user_manual.md
index a4f3d1d..8b8ac7c 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -367,6 +367,7 @@ Isar can generate various images types for specific machine. The type of the ima
  - `rpi-sdimg` - A complete, partitioned Raspberry Pi SD card image (default option for the `rpi` machine).
  - `wic-img` - A full disk image with user-specified partitions created and populated using the wic tool.
  - `ubi-img` - A image for use on mtd nand partitions employing UBI
+ - `virtual-machine-image` - A image for use on VirtualBox or VMware
 
 ---
 
diff --git a/meta-isar/conf/machine/virtualbox.conf b/meta-isar/conf/machine/virtualbox.conf
new file mode 100644
index 0000000..9b823ff
--- /dev/null
+++ b/meta-isar/conf/machine/virtualbox.conf
@@ -0,0 +1,16 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2020
+#
+# SPDX-License-Identifier: MIT
+
+DISTRO_ARCH ?= "amd64"
+
+KERNEL_NAME ?= "amd64"
+BOOTLOADER ?= "grub"
+
+WKS_FILE ?= "sdimage-efi"
+
+IMAGER_INSTALL += "${GRUB_BOOTLOADER_INSTALL}"
+
+VMDK_SUBFORMAT = "monolithicSparse"
+IMAGE_TYPE ?= "virtual-machine-image"
diff --git a/meta-isar/conf/machine/vmware.conf b/meta-isar/conf/machine/vmware.conf
new file mode 100644
index 0000000..b6b4cc7
--- /dev/null
+++ b/meta-isar/conf/machine/vmware.conf
@@ -0,0 +1,16 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2020
+#
+# SPDX-License-Identifier: MIT
+
+DISTRO_ARCH ?= "amd64"
+
+KERNEL_NAME ?= "amd64"
+BOOTLOADER ?= "grub"
+
+WKS_FILE ?= "sdimage-efi"
+
+IMAGER_INSTALL += "${GRUB_BOOTLOADER_INSTALL}"
+
+VMDK_SUBFORMAT = "streamOptimized"
+IMAGE_TYPE ?= "virtual-machine-image"
diff --git a/meta-isar/conf/multiconfig/virtualbox-ova-buster.conf b/meta-isar/conf/multiconfig/virtualbox-ova-buster.conf
new file mode 100644
index 0000000..3042556
--- /dev/null
+++ b/meta-isar/conf/multiconfig/virtualbox-ova-buster.conf
@@ -0,0 +1,8 @@
+#
+# Copyright (c) Siemens AG, 2020
+#
+# SPDX-License-Identifier: MIT
+
+
+MACHINE = "virtualbox"
+DISTRO = "debian-buster"
diff --git a/meta/classes/virtual-machine-image.bbclass b/meta/classes/virtual-machine-image.bbclass
new file mode 100644
index 0000000..10e86a1
--- /dev/null
+++ b/meta/classes/virtual-machine-image.bbclass
@@ -0,0 +1,92 @@
+# This software is a part of ISAR.
+# Copyright (C) 2019-2020 Siemens AG
+#
+# This class allows to generate images for vmware and virtualbox
+#
+
+inherit buildchroot
+inherit wic-img
+IMAGER_BUILD_DEPS += "virtual-machine-template"
+IMAGER_INSTALL += "qemu-utils gawk uuid-runtime virtual-machine-template"
+export OVA_NAME ?= "${IMAGE_FULLNAME}"
+export OVA_MEMORY ?= "8192"
+export OVA_NUMBER_OF_CPU ?= "4"
+export OVA_VRAM ?= "64"
+export OVA_FIRMWARE ?= "efi"
+export OVA_ACPI ?= "true"
+export OVA_3D_ACCEL ?= "false"
+export OVA_CLIPBOARD ?= "bidirectional"
+SOURCE_IMAGE_FILE ?= "${IMAGE_FULLNAME}.wic.img"
+OVA_SHA_ALG ?= "1"
+VIRTUAL_MACHINE_IMAGE_TYPE ?= "vmdk"
+export VIRTUAL_MACHINE_IMAGE_FILE ?= "${IMAGE_FULLNAME}-disk001.${VIRTUAL_MACHINE_IMAGE_TYPE}"
+VIRTUAL_MACHINE_DISK ?= "${PP_DEPLOY}/${VIRTUAL_MACHINE_IMAGE_FILE}"
+# for virtualbox this needs to be monolithicSparse
+# for vmware this needs to be streamOptimized
+#VMDK_SUBFORMAT ?= "streamOptimized"
+export VMDK_SUBFORMAT ?= "monolithicSparse"
+def set_convert_options(d):
+   format = d.getVar("VIRTUAL_MACHINE_IMAGE_TYPE")
+   if format == "vmdk":
+      return "-o subformat=%s" % d.getVar("VMDK_SUBFORMAT")
+   else:
+      return ""
+
+
+CONVERSION_OPTIONS = "${@set_convert_options(d)}"
+
+do_convert_wic() {
+   rm -f '${DEPLOY_DIR_IMAGE}/${VIRTUAL_MACHINE_IMAGE_FILE}'
+   image_do_mounts
+   bbnote "Creating ${VIRTUAL_MACHINE_IMAGE_FILE} from ${WIC_IMAGE_FILE}"
+   sudo -E  chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+   /usr/bin/qemu-img convert -f raw -O ${VIRTUAL_MACHINE_IMAGE_TYPE} ${CONVERSION_OPTIONS} \
+       '${PP_DEPLOY}/${SOURCE_IMAGE_FILE}' '${PP_DEPLOY}/${VIRTUAL_MACHINE_IMAGE_FILE}'
+}
+
+addtask convert_wic before do_build after do_wic_image do_copy_boot_files do_install_imager_deps do_transform_template
+
+# Generate random MAC addresses just as VirtualBox does, the format is
+# their assigned prefix for the first 3 bytes followed by 3 random bytes.
+VBOX_MAC_PREFIX = "080027"
+macgen() {
+    hexdump -n3 -e "\"${VBOX_MAC_PREFIX}%06X\n\"" /dev/urandom
+}
+get_disksize() {
+    image_do_mounts
+    sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+        qemu-img info -f vmdk "${VIRTUAL_MACHINE_DISK}" | gawk 'match($0, /^virtual size:.*\(([0-9]+) bytes\)/, a) {print a[1]}'
+}
+do_create_ova() {
+    if [ ! ${VIRTUAL_MACHINE_IMAGE_TYPE} = "vmdk" ]; then
+        exit 0
+    fi
+    rm -f '${DEPLOY_DIR_IMAGE}/${OVA_NAME}.ova'
+    rm -f '${DEPLOY_DIR_IMAGE}/${OVA_NAME}.ovf'
+    rm -f '${DEPLOY_DIR_IMAGE}/${OVA_NAME}.mf'
+
+    export PRIMARY_MAC=$(macgen)
+    export SECONDARY_MAC=$(macgen)
+    export DISK_NAME=$(basename -s .vmdk ${VIRTUAL_MACHINE_DISK})
+    export DISK_SIZE_BYTES=$(get_disksize)
+    export LAST_CHANGE=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
+    export OVA_FIRMWARE_VIRTUALBOX=$(echo ${OVA_FIRMWARE} | tr '[a-z]' '[A-Z]')
+    image_do_mounts
+    sudo -Es chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} <<'EOSUDO'
+        export DISK_UUID=$(uuidgen)
+        export VM_UUID=$(uuidgen)
+        # create ovf
+        cat /usr/share/virtual-machine-template/virtual-machine-template.ovf.tmpl | envsubst > ${PP_DEPLOY}/${OVA_NAME}.ovf
+        tar -cvf ${PP_DEPLOY}/${OVA_NAME}.ova -C ${PP_DEPLOY} ${OVA_NAME}.ovf
+
+        # virtual box needs here a manifest file vmware does not want to accept the format
+        if [ "${VMDK_SUBFORMAT}" = "monolithicSparse" ]; then
+            echo "SHA${OVA_SHA_ALG}(${VIRTUAL_MACHINE_IMAGE_FILE})=$(sha${OVA_SHA_ALG}sum ${PP_DEPLOY}/${VIRTUAL_MACHINE_IMAGE_FILE} | cut -d' ' -f1)" >> ${PP_DEPLOY}/${OVA_NAME}.mf
+            echo "SHA${OVA_SHA_ALG}(${OVA_NAME}.ovf)=$(sha${OVA_SHA_ALG}sum ${PP_DEPLOY}/${OVA_NAME}.ovf | cut -d' ' -f1)" >> ${PP_DEPLOY}/${OVA_NAME}.mf
+            tar -uvf ${PP_DEPLOY}/${OVA_NAME}.ova -C ${PP_DEPLOY} ${OVA_NAME}.mf
+        fi
+        tar -uvf ${PP_DEPLOY}/${OVA_NAME}.ova -C ${PP_DEPLOY} ${VIRTUAL_MACHINE_IMAGE_FILE}
+EOSUDO
+}
+
+addtask do_create_ova after do_convert_wic before do_deploy
diff --git a/meta/recipes-devtools/virtual-machine-template/files/virtual-machine-template.ovf.tmpl b/meta/recipes-devtools/virtual-machine-template/files/virtual-machine-template.ovf.tmpl
new file mode 100644
index 0000000..e6b5305
--- /dev/null
+++ b/meta/recipes-devtools/virtual-machine-template/files/virtual-machine-template.ovf.tmpl
@@ -0,0 +1,155 @@
+<?xml version="1.0"?>
+<Envelope ovf:version="1.0" xml:lang="en-US" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:vbox="http://www.virtualbox.org/ovf/machine">
+  <References>
+    <File ovf:href="${VIRTUAL_MACHINE_IMAGE_FILE}" ovf:id="file1"/>
+  </References>
+  <DiskSection>
+    <Info>List of the virtual disks used in the package</Info>
+    <Disk ovf:capacity="${DISK_SIZE_BYTES}" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#${VMDK_SUBFORMAT}" vbox:uuid="${DISK_UUID}"/>
+  </DiskSection>
+  <NetworkSection>
+    <Info>Logical networks used in the package</Info>
+    <Network ovf:name="NAT">
+      <Description>Logical network used by this appliance.</Description>
+    </Network>
+  </NetworkSection>
+  <VirtualSystem ovf:id="${OVA_NAME}">
+    <Info>A virtual machine</Info>
+    <OperatingSystemSection ovf:id="96">
+      <Info>The kind of installed guest operating system</Info>
+      <Description>Debian_64</Description>
+      <vbox:OSType ovf:required="false">Debian_64</vbox:OSType>
+    </OperatingSystemSection>
+    <VirtualHardwareSection>
+      <Info>Virtual hardware requirements for a virtual machine</Info>
+      <System>
+        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
+        <vssd:InstanceID>0</vssd:InstanceID>
+        <vssd:VirtualSystemIdentifier>${OVA_NAME}</vssd:VirtualSystemIdentifier>
+        <vssd:VirtualSystemType>virtualbox-2.2</vssd:VirtualSystemType>
+      </System>
+      <Item>
+        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
+        <rasd:Caption>${OVA_NUMBER_OF_CPU} virtual CPU</rasd:Caption>
+        <rasd:Description>Number of virtual CPUs</rasd:Description>
+        <rasd:ElementName>${OVA_NUMBER_OF_CPU} virtual CPU</rasd:ElementName>
+        <rasd:InstanceID>1</rasd:InstanceID>
+        <rasd:ResourceType>3</rasd:ResourceType>
+        <rasd:VirtualQuantity>${OVA_NUMBER_OF_CPU}</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:AllocationUnits>MegaBytes</rasd:AllocationUnits>
+        <rasd:Caption>${OVA_MEMORY} MB of memory</rasd:Caption>
+        <rasd:Description>Memory Size</rasd:Description>
+        <rasd:ElementName>${OVA_MEMORY} MB of memory</rasd:ElementName>
+        <rasd:InstanceID>2</rasd:InstanceID>
+        <rasd:ResourceType>4</rasd:ResourceType>
+        <rasd:VirtualQuantity>${OVA_MEMORY}</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:Address>0</rasd:Address>
+        <rasd:Caption>ideController0</rasd:Caption>
+        <rasd:Description>IDE Controller</rasd:Description>
+        <rasd:ElementName>ideController0</rasd:ElementName>
+        <rasd:InstanceID>3</rasd:InstanceID>
+        <rasd:ResourceSubType>PIIX4</rasd:ResourceSubType>
+        <rasd:ResourceType>5</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:Address>1</rasd:Address>
+        <rasd:Caption>ideController1</rasd:Caption>
+        <rasd:Description>IDE Controller</rasd:Description>
+        <rasd:ElementName>ideController1</rasd:ElementName>
+        <rasd:InstanceID>4</rasd:InstanceID>
+        <rasd:ResourceSubType>PIIX4</rasd:ResourceSubType>
+        <rasd:ResourceType>5</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
+        <rasd:Caption>Ethernet adapter on 'NAT'</rasd:Caption>
+        <rasd:Connection>NAT</rasd:Connection>
+        <rasd:ElementName>Ethernet adapter on 'NAT'</rasd:ElementName>
+        <rasd:InstanceID>5</rasd:InstanceID>
+        <rasd:ResourceSubType>E1000</rasd:ResourceSubType>
+        <rasd:ResourceType>10</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:Caption>disk1</rasd:Caption>
+        <rasd:Description>Disk Image</rasd:Description>
+        <rasd:ElementName>disk1</rasd:ElementName>
+        <rasd:HostResource>/disk/vmdisk1</rasd:HostResource>
+        <rasd:InstanceID>6</rasd:InstanceID>
+        <rasd:Parent>3</rasd:Parent>
+        <rasd:ResourceType>17</rasd:ResourceType>
+      </Item>
+      <vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="${OVA_FIRMWARE}"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true"/>
+      <vmw:ExtraConfig ovf:required="false" vmw:key="virtualHW.productCompatibility" vmw:value="hosted"/>
+      </VirtualHardwareSection>
+      <vbox:Machine ovf:required="false" version="1.12-linux" uuid="{${VM_UUID}}" name="${OVA_NAME}" OSType="Debian_64" snapshotFolder="Snapshots" lastStateChange="${LAST_CHANGE}">
+        <ovf:Info>Complete VirtualBox machine configuration in VirtualBox format</ovf:Info>
+        <Hardware>
+          <CPU count="${OVA_NUMBER_OF_CPU}">
+            <PAE enabled="true"/>
+            <HardwareVirtExLargePages enabled="false"/>
+          </CPU>
+          <Memory RAMSize="${OVA_MEMORY}"/>
+          <Firmware type="${OVA_FIRMWARE_VIRTUALBOX}"/>
+          <Boot>
+            <Order position="1" device="HardDisk"/>
+            <Order position="2" device="None"/>
+            <Order position="3" device="None"/>
+            <Order position="4" device="None"/>
+          </Boot>
+          <Display VRAMSize="${OVA_VRAM}" monitorCount="1" accelerate3D="${OVA_3D_ACCEL}" accelerate2DVideo="false"/>
+          <VideoRecording enabled="false" file="Test.webm" horzRes="640" vertRes="480"/>
+          <RemoteDisplay enabled="false" authType="Null"/>
+          <BIOS>
+            <IOAPIC enabled="${OVA_ACPI}"/>
+          </BIOS>
+          <USBController enabled="false" enabledEhci="false"/>
+          <Network>
+            <Adapter slot="0" enabled="true" MACAddress="${PRIMARY_MAC}" cable="true" speed="0" type="virtio">
+              <DisabledModes/>
+              <NAT>
+                <DNS pass-domain="true" use-proxy="false" use-host-resolver="false"/>
+                <Alias logging="false" proxy-only="false" use-same-ports="false"/>
+              </NAT>
+            </Adapter>
+          </Network>
+          <LPT>
+            <Port slot="1" enabled="false" IOBase="0x378" IRQ="7"/>
+          </LPT>
+          <AudioAdapter driver="Pulse" enabled="false"/>
+          <RTC localOrUTC="local"/>
+          <SharedFolders/>
+          <Clipboard mode="Disabled"/>
+          <DragAndDrop mode="Disabled"/>
+          <IO>
+            <IoCache enabled="true" size="5"/>
+            <BandwidthGroups/>
+          </IO>
+          <HostPci>
+            <Devices/>
+          </HostPci>
+          <EmulatedUSB>
+            <CardReader enabled="false"/>
+          </EmulatedUSB>
+          <Guest memoryBalloonSize="0"/>
+          <GuestProperties/>
+        </Hardware>
+        <StorageControllers>
+          <StorageController name="IDE Controller" type="PIIX4" PortCount="2" useHostIOCache="true" Bootable="true">
+            <AttachedDevice type="HardDisk" port="0" device="0">
+              <Image uuid="{${DISK_UUID}}"/>
+            </AttachedDevice>
+          </StorageController>
+        </StorageControllers>
+      </vbox:Machine>
+  </VirtualSystem>
+</Envelope>
diff --git a/meta/recipes-devtools/virtual-machine-template/virtual-machine-template_0.1.bb b/meta/recipes-devtools/virtual-machine-template/virtual-machine-template_0.1.bb
new file mode 100644
index 0000000..dc67355
--- /dev/null
+++ b/meta/recipes-devtools/virtual-machine-template/virtual-machine-template_0.1.bb
@@ -0,0 +1,17 @@
+# This software is a part of ISAR.
+#
+# Copyright (c) Siemens AG, 2020
+#
+# SPDX-License-Identifier: MIT
+
+inherit dpkg-raw
+
+
+SRC_URI += "file://virtual-machine-template.ovf.tmpl"
+
+do_install() {
+    TARGET=${D}/usr/share/virtual-machine-template
+    install -m 0755 -d ${TARGET}
+    install -m 0740 ${WORKDIR}/virtual-machine-template.ovf.tmpl \
+        ${TARGET}/virtual-machine-template.ovf.tmpl
+}
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index a8d9ad9..005bda5 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -32,7 +32,8 @@ TARGETS_SET="\
             mc:qemumipsel-buster:isar-image-base \
             mc:nand-ubi-demo-buster:isar-image-ubi \
             mc:rpi-stretch:isar-image-base \
-            mc:qemuamd64-focal:isar-image-base"
+            mc:qemuamd64-focal:isar-image-base \
+            mc:virtualbox-ova-buster:isar-image-base"
           # qemu-user-static of <= buster too old to build that
           # mc:qemuarm64-buster:isar-image-base
           # mc:qemuarm64-bullseye:isar-image-base
-- 
2.20.1