Re: [RFC][PATCH 1/6] base-apt: Introduce fetching upstream apt

[Jan Kiszka <jan.kiszka@siemens.com>]

On 2018-01-02 15:57, Alexander Smirnov wrote:
> This patch introduces mechanism how to fetch deb packages from dedicated
> upstream apt repos and store them localy. Local repository is called
> 'base-apt' and it will be used to generate buildchroot and image root
> filesystems. Using 'base-apt' will guarantee build reproducibility between
> builds.
> 
> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
> ---
>  meta/conf/isar-bitbake.conf                        |  2 +
>  meta/recipes-devtools/base-apt/base-apt.bb         | 79 ++++++++++++++++++++++
>  .../base-apt/files/distributions.in                |  3 +
>  .../base-apt/files/multistrap.conf.in              | 28 ++++++++
>  4 files changed, 112 insertions(+)
>  create mode 100644 meta/recipes-devtools/base-apt/base-apt.bb
>  create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
>  create mode 100644 meta/recipes-devtools/base-apt/files/multistrap.conf.in
> 
> diff --git a/meta/conf/isar-bitbake.conf b/meta/conf/isar-bitbake.conf
> index 5a26743..df54399 100644
> --- a/meta/conf/isar-bitbake.conf
> +++ b/meta/conf/isar-bitbake.conf
> @@ -23,6 +23,8 @@ DEPLOY_DIR_DEB = "${TMPDIR}/deploy/deb/${MACHINE}"
>  SSTATE_DIR ?= "${TMPDIR}/sstate-cache"
>  BUILDCHROOT_DIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/buildchroot/rootfs"
>  
> +BASE_APT_DIR ?= "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/base-apt"

That should also go into TOPDIR, analogously to my patch for DL_DIR.

> +
>  # Setup our default hash policy
>  BB_SIGNATURE_HANDLER ?= "noop"
>  
> diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
> new file mode 100644
> index 0000000..b292d89
> --- /dev/null
> +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> @@ -0,0 +1,79 @@
> +# Caching upstream apt repository to local one.
> +#
> +# This software is a part of ISAR.
> +# Copyright (C) 2015-2017 ilbers GmbH
> +
> +DESCRIPTION = "Upstream apt caching"
> +
> +LICENSE = "gpl-2.0"
> +LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
> +
> +FILESPATH =. "${LAYERDIR_core}/recipes-devtools/base-apt/files:"
> +SRC_URI = "file://distributions.in \
> +           file://multistrap.conf.in \
> +          "
> +
> +BASE_PREINSTALL ?= ""
> +
> +WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
> +
> +BASE_APT_CONF_DIR = "${BASE_APT_DIR}/apt/conf"
> +do_get_base_apt[dirs] = "${BASE_APT_CONF_DIR}"
> +do_get_base_apt[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
> +
> +do_get_base_apt() {
> +    # Adjust multistrap config
> +    sed -e 's|##BASE_PREINSTALL##|${BASE_PREINSTALL}|g' \
> +        -e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
> +        -e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
> +        -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
> +        -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
> +        -e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
> +        -e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
> +        -e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/configscript.sh|g' \
> +        -e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
> +        -e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
> +           "${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
> +
> +    # Fetch deb packages
> +    sudo -E multistrap \
> +        -a ${DISTRO_ARCH} \
> +        -d "${WORKDIR}/download" \
> +        -f "${WORKDIR}/multistrap.conf" \
> +        > ${WORKDIR}/multistrap.log 2>&1
> +
> +    if [ ! -e "${BASE_APT_CONF_DIR}/distributions" ]; then
> +        sed -e "s#{DISTRO_NAME}#"${DISTRO_SUITE}"#g" \
> +            ${WORKDIR}/distributions.in > ${BASE_APT_CONF_DIR}/distributions
> +    fi
> +
> +    # Create reprepro cache
> +    if [ ! -d "${BASE_APT_DIR}/apt" ]; then
> +        reprepro -b ${BASE_APT_DIR}/apt \
> +                 --dbdir ${BASE_APT_DIR}/db \
> +                 export ${DISTRO_SUITE}
> +    fi
> +
> +    # Process all the packages fetched by multistrap
> +    for deb in $(ls ${WORKDIR}/download/var/cache/apt/archives/*.deb);
> +    do
> +        pn=$(dpkg-deb -I $deb | grep 'Package:' | cut -d ' ' -f 3)
> +        pv=$(dpkg-deb -I $deb | grep 'Version:' | cut -d ' ' -f 3)
> +        line=$(cat ${WORKDIR}/multistrap.log | grep -E '^(Get:)' | grep " $pn ")
> +        url=$(echo $line | cut -d ' ' -f 2)
> +        component=$(basename $(echo $line | cut -d ' ' -f 3))
> +
> +        # Store download history
> +        echo $pn $pv $component $url >> ${WORKDIR}/deb.list
> +
> +        reprepro -b ${BASE_APT_DIR}/apt \
> +                 --dbdir ${BASE_APT_DIR}/db \
> +                 -C $component \
> +                 includedeb ${DISTRO_SUITE} \
> +                 $deb
> +    done
> +
> +    sudo rm -rf ${WORKDIR}/download
> +}
> +
> +addtask get_base_apt after do_unpack before do_build
> diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
> new file mode 100644
> index 0000000..44e9513
> --- /dev/null
> +++ b/meta/recipes-devtools/base-apt/files/distributions.in
> @@ -0,0 +1,3 @@
> +Codename: {DISTRO_NAME}
> +Architectures: i386 armhf amd64 source
> +Components: main contrib non-free firmware
> diff --git a/meta/recipes-devtools/base-apt/files/multistrap.conf.in b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
> new file mode 100644
> index 0000000..27bf985
> --- /dev/null
> +++ b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
> @@ -0,0 +1,28 @@
> +# This software is a part of ISAR.
> +# Copyright (C) 2015-2017 ilbers GmbH
> +
> +[General]
> +noauth=true
> +unpack=false
> +ignorenativearch=true
> +bootstrap=##DISTRO_MULTICONF_BOOTSTRAP##
> +aptsources=##DISTRO_MULTICONF_APTSOURCES##
> +
> +[base]
> +source=##DISTRO_APT_SOURCE##
> +suite=##DISTRO_SUITE##
> +components=##DISTRO_COMPONENTS##
> +packages=##BASE_PREINSTALL##
> +omitdebsrc=true
> +
> +[updates]
> +source=##DISTRO_APT_SOURCE##
> +suite=##DISTRO_SUITE##-updates
> +components=##DISTRO_COMPONENTS##
> +omitdebsrc=true
> +
> +[security]
> +source=##DISTRO_APT_SOURCE_SEC##
> +suite=##DISTRO_SUITE##/updates
> +components=##DISTRO_COMPONENTS##
> +omitdebsrc=true
> 

Hmm, is there really no way to share the common bits with
meta/recipes-devtools/buildchroot/files/multistrap.conf.in? I suspect
the want to be kept in sync, except for the General section.

Jan

-- 
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux