Re: [PATCH 1/2] sdk: support creation of container image

[Silvano Cirujano Cuesta <silvano.cirujano-cuesta@siemens.com>]

On 05/02/2021 12:29, Jan Kiszka wrote:
> On 05.02.21 12:24, Silvano Cirujano Cuesta wrote:
>> On 05/02/2021 12:07, Jan Kiszka wrote:
>>> On 05.02.21 10:08, [ext] Silvano Cirujano Cuesta wrote:
>>>> Extend task "populate_sdk" to support the creation of a container image
>>>> containing the SDK.
>>>>
>>>> Signed-off-by: Silvano Cirujano Cuesta <silvano.cirujano-cuesta@siemens.com>
>>>> ---
>>>>  meta/classes/image-sdk-extension.bbclass | 104 +++++++++++++++++++++--
>>>>  1 file changed, 97 insertions(+), 7 deletions(-)
>>>>
>>>> diff --git a/meta/classes/image-sdk-extension.bbclass b/meta/classes/image-sdk-extension.bbclass
>>>> index a8c708a..082b16d 100644
>>>> --- a/meta/classes/image-sdk-extension.bbclass
>>>> +++ b/meta/classes/image-sdk-extension.bbclass
>>>> @@ -6,10 +6,81 @@
>>>>  # This class extends the image.bbclass to supply the creation of a sdk
>>>>  
>>>>  SDK_INCLUDE_ISAR_APT ?= "0"
>>>> +SDK_FORMATS ?= "tar-xz"
>>>> +
>>>> +sdk_tar_xz() {
>>>> +    # Copy mount_chroot.sh for convenience
>>>> +    sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR}
>>>> +
>>>> +    # Create SDK archive
>>>> +    cd -P ${SDKCHROOT_DIR}/..
>>>> +    sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \
>>>> +        -c rootfs | xz -T0 > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz
>>>> +    bbnote "SDK rootfs available in ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz"
>>>> +}
>>>> +
>>>> +sdk_container_images() {
>>>> +    local cmd="/bin/dash"
>>>> +    local empty_tag="empty"
>>>> +    local full_tag="latest"
>>>> +    local oci_img_dir="${WORKDIR}/oci-image"
>>>> +    local sdk_container_formats="$1"
>>>> +
>>>> +    # prepare OCI container image skeleton
>>>> +    sudo umoci init --layout "${oci_img_dir}"
>>>> +    sudo umoci new --image "${oci_img_dir}:${empty_tag}"
>>>> +    sudo umoci config --image "${oci_img_dir}:${empty_tag}" \
>>>> +        --config.cmd="${cmd}"
>>>> +    sudo umoci unpack --image "${oci_img_dir}:${empty_tag}" \
>>>> +        "${oci_img_dir}_unpacked"
>>>> +
>>>> +    # add SDK root filesystem as the flesh of the skeleton
>>>> +    sudo cp -a "${SDKCHROOT_DIR}"/* "${oci_img_dir}_unpacked/rootfs/"
>>>> +
>>>> +    # pack container image
>>>> +    sudo umoci repack --image "${oci_img_dir}:${full_tag}" \
>>>> +        "${oci_img_dir}_unpacked"
>>>> +    sudo umoci remove --image "${oci_img_dir}:${empty_tag}"
>>>> +    sudo rm -rf "${oci_img_dir}_unpacked"
>>>> +
>>>> +    # no root needed anymore
>>>> +    sudo chown --recursive $(id -u):$(id -g) "${oci_img_dir}"
>>>> +
>>>> +    # convert the OCI container image to the desired format
>>>> +    sdk_id="sdk-${DISTRO}-${DISTRO_ARCH}"
>>>> +    image_name="isar-${sdk_id}"
>>>> +    image_archive="${DEPLOY_DIR_IMAGE}/${sdk_id}-${sdk_format}.tar"
>>>> +    for sdk_format in ${sdk_container_formats} ; do
>>>> +        case "${sdk_format}" in
>>>> +            "docker-archive" | "oci-archive")
>>>> +                if [ "${sdk_format}" = "oci-archive" ] ; then
>>>> +                    target="${sdk_format}:${image_archive}:latest"
>>>> +                else
>>>> +                    target="${sdk_format}:${image_archive}:${image_name}:latest"
>>>> +                fi
>>>> +                skopeo --insecure-policy copy \
>>>> +                    "oci:${oci_img_dir}:${full_tag}" "${target}"
>>>> +                xz -T0 "${image_archive}"
>>>> +                bbnote "Containerized SDK available in ${image_archive}.xz"
>>>> +                ;;
>>>> +            "oci")
>>>> +                tar --create --xz --directory "${oci_img_dir}" \
>>>> +                    --file "${image_archive}.xz" .
>>>> +                bbnote "Containerized SDK available in ${image_archive}.xz"
>>>> +                ;;
>>>> +            "docker-daemon" | "containers-storage")
>>>> +                skopeo --insecure-policy copy \
>>>> +                    "oci:${oci_img_dir}:${full_tag}" \
>>>> +                    "${sdk_format}:${image_name}:latest"
>>>> +                bbnote "Containerized SDK available in ${sdk_format} as '${image_name}:latest'"
>>>> +                ;;
>>>> +        esac
>>>> +    done
>>>> +}
>>>>  
>>>>  do_populate_sdk[stamp-extra-info] = "${DISTRO}-${MACHINE}"
>>>>  do_populate_sdk[depends] = "sdkchroot:do_build"
>>>> -do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT"
>>>> +do_populate_sdk[vardeps] += "SDK_INCLUDE_ISAR_APT SDK_FORMATS"
>>>>  do_populate_sdk() {
>>>>      if [ "${SDK_INCLUDE_ISAR_APT}" = "1" ]; then
>>>>          # Copy isar-apt with deployed Isar packages
>>>> @@ -48,12 +119,31 @@ do_populate_sdk() {
>>>>          done
>>>>      done
>>>>  
>>>> -    # Copy mount_chroot.sh for convenience
>>>> -    sudo cp ${SCRIPTSDIR}/mount_chroot.sh ${SDKCHROOT_DIR}
>>>> +    # separate SDK formats: TAR and container formats
>>>> +    container_formats=""
>>>> +    for sdk_format in ${SDK_FORMATS} ; do
>>>> +        case ${sdk_format} in
>>>> +            "tar-xz")
>>>> +                sdk_tar_xz
>>>> +                ;;
>>>> +            "docker-archive" | "oci" | "oci-archive")
>>>> +                container_formats="${container_formats} ${sdk_format}"
>>>> +                ;;
>>>> +            "docker-daemon" | "containers-storage")
>>>> +                if [ -f /.dockerenv ] || [ -f /run/.containerenv ] ; then
>>>> +                    die "Adding the SDK container image to a container runtime (${sdk_format}) not supported if running from a container (e.g. 'kas-container')"
>>>> +                fi
>>>> +                ;;
>>>> +            *)
>>>> +                die "unsupported SDK format specified: ${sdk_format}"
>>>> +                ;;
>>>> +        esac
>>>> +    done
>>>>  
>>>> -    # Create SDK archive
>>>> -    cd -P ${SDKCHROOT_DIR}/..
>>>> -    sudo tar --transform="s|^rootfs|sdk-${DISTRO}-${DISTRO_ARCH}|" \
>>>> -        -c rootfs | xz -T0 > ${DEPLOY_DIR_IMAGE}/sdk-${DISTRO}-${DISTRO_ARCH}.tar.xz
>>>> +    # generate the SDK in all the desired container formats
>>>> +    if [ -n "${container_formats}" ] ; then
>>>> +        bbnote "Generating SDK container in${container_formats} format"
>>>> +        sdk_container_images "${container_formats}"
>>>> +    fi
>>>>  }
>>>>  addtask populate_sdk after do_rootfs
>>>>
>>> How much of this would be reusable of generating a container from a
>>> target rootfs? We should avoid shuffling code around if we can already
>>> line things up nicely while introducing it.
>> With that reuse in mind I can refactor the code to have a class that can be reused for both SDK and target rootfs. It's not a big deal.
>>
>> Is it somehow related to the huge "some image classes" thread that is active right now?
>>
> Not directly related, but container images would fall into that same
> category.

What do you have in mind? How should I structure the functions, classes, tasks,...?

In a separate bbclass providing only generic functions that can containerize whatever rootfs (e.g. target) they're given (currently only for the SDK calling them from "populate_sdk")? Similar to "debianize.bbclass". Something I can do easily.

In a separate bbclass following the pattern of the other "-img.bclass" (e.g. "container-img.bbclass")? I mean providing not only functions even new tasks. Would be harder to accomplish.

Are the "-img.bbclass" classes only for target images? I wonder why "targz-img.bbclass" doesn't take care of "packaging" also the SDK. Not because of code reuse (too short code, to many differences on the TAR call), but for logical consistency.

  Silvano
>
> Jan
>
-- 
Siemens AG, T RDA IOT SES-DE
Corporate Competence Center Embedded Linux