Re: [RFC][PATCH 1/6] base-apt: Introduce fetching upstream apt

[Alexander Smirnov <asmirnov@ilbers.de>]

Hi,

On 01/02/2018 07:15 PM, Jan Kiszka wrote:
> On 2018-01-02 15:57, Alexander Smirnov wrote:
>> This patch introduces mechanism how to fetch deb packages from dedicated
>> upstream apt repos and store them localy. Local repository is called
>> 'base-apt' and it will be used to generate buildchroot and image root
>> filesystems. Using 'base-apt' will guarantee build reproducibility between
>> builds.
>>
>> Signed-off-by: Alexander Smirnov <asmirnov@ilbers.de>
>> ---
>>   meta/conf/isar-bitbake.conf                        |  2 +
>>   meta/recipes-devtools/base-apt/base-apt.bb         | 79 ++++++++++++++++++++++
>>   .../base-apt/files/distributions.in                |  3 +
>>   .../base-apt/files/multistrap.conf.in              | 28 ++++++++
>>   4 files changed, 112 insertions(+)
>>   create mode 100644 meta/recipes-devtools/base-apt/base-apt.bb
>>   create mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
>>   create mode 100644 meta/recipes-devtools/base-apt/files/multistrap.conf.in
>>
>> diff --git a/meta/conf/isar-bitbake.conf b/meta/conf/isar-bitbake.conf
>> index 5a26743..df54399 100644
>> --- a/meta/conf/isar-bitbake.conf
>> +++ b/meta/conf/isar-bitbake.conf
>> @@ -23,6 +23,8 @@ DEPLOY_DIR_DEB = "${TMPDIR}/deploy/deb/${MACHINE}"
>>   SSTATE_DIR ?= "${TMPDIR}/sstate-cache"
>>   BUILDCHROOT_DIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/buildchroot/rootfs"
>>   
>> +BASE_APT_DIR ?= "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/base-apt"
> 
> That should also go into TOPDIR, analogously to my patch for DL_DIR.
> 

No problem, have included this to fix-list for next series version.

>> +
>>   # Setup our default hash policy
>>   BB_SIGNATURE_HANDLER ?= "noop"
>>   
>> diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
>> new file mode 100644
>> index 0000000..b292d89
>> --- /dev/null
>> +++ b/meta/recipes-devtools/base-apt/base-apt.bb
>> @@ -0,0 +1,79 @@
>> +# Caching upstream apt repository to local one.
>> +#
>> +# This software is a part of ISAR.
>> +# Copyright (C) 2015-2017 ilbers GmbH
>> +
>> +DESCRIPTION = "Upstream apt caching"
>> +
>> +LICENSE = "gpl-2.0"
>> +LIC_FILES_CHKSUM = "file://${LAYERDIR_isar}/licenses/COPYING.GPLv2;md5=751419260aa954499f7abaabaa882bbe"
>> +
>> +FILESPATH =. "${LAYERDIR_core}/recipes-devtools/base-apt/files:"
>> +SRC_URI = "file://distributions.in \
>> +           file://multistrap.conf.in \
>> +          "
>> +
>> +BASE_PREINSTALL ?= ""
>> +
>> +WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
>> +
>> +BASE_APT_CONF_DIR = "${BASE_APT_DIR}/apt/conf"
>> +do_get_base_apt[dirs] = "${BASE_APT_CONF_DIR}"
>> +do_get_base_apt[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
>> +
>> +do_get_base_apt() {
>> +    # Adjust multistrap config
>> +    sed -e 's|##BASE_PREINSTALL##|${BASE_PREINSTALL}|g' \
>> +        -e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
>> +        -e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
>> +        -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
>> +        -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
>> +        -e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
>> +        -e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
>> +        -e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/configscript.sh|g' \
>> +        -e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
>> +        -e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
>> +           "${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
>> +
>> +    # Fetch deb packages
>> +    sudo -E multistrap \
>> +        -a ${DISTRO_ARCH} \
>> +        -d "${WORKDIR}/download" \
>> +        -f "${WORKDIR}/multistrap.conf" \
>> +        > ${WORKDIR}/multistrap.log 2>&1
>> +
>> +    if [ ! -e "${BASE_APT_CONF_DIR}/distributions" ]; then
>> +        sed -e "s#{DISTRO_NAME}#"${DISTRO_SUITE}"#g" \
>> +            ${WORKDIR}/distributions.in > ${BASE_APT_CONF_DIR}/distributions
>> +    fi
>> +
>> +    # Create reprepro cache
>> +    if [ ! -d "${BASE_APT_DIR}/apt" ]; then
>> +        reprepro -b ${BASE_APT_DIR}/apt \
>> +                 --dbdir ${BASE_APT_DIR}/db \
>> +                 export ${DISTRO_SUITE}
>> +    fi
>> +
>> +    # Process all the packages fetched by multistrap
>> +    for deb in $(ls ${WORKDIR}/download/var/cache/apt/archives/*.deb);
>> +    do
>> +        pn=$(dpkg-deb -I $deb | grep 'Package:' | cut -d ' ' -f 3)
>> +        pv=$(dpkg-deb -I $deb | grep 'Version:' | cut -d ' ' -f 3)
>> +        line=$(cat ${WORKDIR}/multistrap.log | grep -E '^(Get:)' | grep " $pn ")
>> +        url=$(echo $line | cut -d ' ' -f 2)
>> +        component=$(basename $(echo $line | cut -d ' ' -f 3))
>> +
>> +        # Store download history
>> +        echo $pn $pv $component $url >> ${WORKDIR}/deb.list
>> +
>> +        reprepro -b ${BASE_APT_DIR}/apt \
>> +                 --dbdir ${BASE_APT_DIR}/db \
>> +                 -C $component \
>> +                 includedeb ${DISTRO_SUITE} \
>> +                 $deb
>> +    done
>> +
>> +    sudo rm -rf ${WORKDIR}/download
>> +}
>> +
>> +addtask get_base_apt after do_unpack before do_build
>> diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
>> new file mode 100644
>> index 0000000..44e9513
>> --- /dev/null
>> +++ b/meta/recipes-devtools/base-apt/files/distributions.in
>> @@ -0,0 +1,3 @@
>> +Codename: {DISTRO_NAME}
>> +Architectures: i386 armhf amd64 source
>> +Components: main contrib non-free firmware
>> diff --git a/meta/recipes-devtools/base-apt/files/multistrap.conf.in b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
>> new file mode 100644
>> index 0000000..27bf985
>> --- /dev/null
>> +++ b/meta/recipes-devtools/base-apt/files/multistrap.conf.in
>> @@ -0,0 +1,28 @@
>> +# This software is a part of ISAR.
>> +# Copyright (C) 2015-2017 ilbers GmbH
>> +
>> +[General]
>> +noauth=true
>> +unpack=false
>> +ignorenativearch=true
>> +bootstrap=##DISTRO_MULTICONF_BOOTSTRAP##
>> +aptsources=##DISTRO_MULTICONF_APTSOURCES##
>> +
>> +[base]
>> +source=##DISTRO_APT_SOURCE##
>> +suite=##DISTRO_SUITE##
>> +components=##DISTRO_COMPONENTS##
>> +packages=##BASE_PREINSTALL##
>> +omitdebsrc=true
>> +
>> +[updates]
>> +source=##DISTRO_APT_SOURCE##
>> +suite=##DISTRO_SUITE##-updates
>> +components=##DISTRO_COMPONENTS##
>> +omitdebsrc=true
>> +
>> +[security]
>> +source=##DISTRO_APT_SOURCE_SEC##
>> +suite=##DISTRO_SUITE##/updates
>> +components=##DISTRO_COMPONENTS##
>> +omitdebsrc=true
>>
> 
> Hmm, is there really no way to share the common bits with
> meta/recipes-devtools/buildchroot/files/multistrap.conf.in? I suspect
> the want to be kept in sync, except for the General section.

multistrap config for buildchroot and image will not use these sections 
anymore, they are dropped later in this series.

Alex