Re: [PATCH 1/5] base: add populate_repo task to include distro packages to the repo

[Alexander Smirnov <asmirnov@ilbers.de>]

Hi,

n 02/04/2018 08:54 PM, Cedric_Hombourger@mentor.com wrote:
> From: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> 
> A package repository is already created for custom packages (built by
> Isar). Provide a populate_repo task to also include any packages
> included into images built by Isar. This mechanism will add both binary
> and source packages. The augmented repository may be used later for
> reproducible builds (technically, packages pulled for the buildchroots
> we build should also be added; this will be the subject of another
> changeset).

Isar-apt should never be used to mirror upstream repositories. Isar-apt 
intended to be an overlay for pure Debian.

Regarding reproducibility, there are already two series discusses in 
this list which introduce dedicated base-apt repository for caching 
upstream:

https://groups.google.com/d/msg/isar-users/9b0f6LP4X6k/LJ3mLxE1EAAJ
https://groups.google.com/d/msg/isar-users/QQUsVmSaAGk/B6FGnVWhAQAJ

Alex

> 
> Signed-off-by: Cedric Hombourger <Cedric_Hombourger@mentor.com>
> ---
>   .../recipes-core/images/files/multistrap.conf.in   |  6 +-
>   meta-isar/recipes-core/images/isar-image-base.bb   | 78 ++++++++++++++++++++++
>   2 files changed, 81 insertions(+), 3 deletions(-)
> 
> diff --git a/meta-isar/recipes-core/images/files/multistrap.conf.in b/meta-isar/recipes-core/images/files/multistrap.conf.in
> index 432b6af..8e4e276 100644
> --- a/meta-isar/recipes-core/images/files/multistrap.conf.in
> +++ b/meta-isar/recipes-core/images/files/multistrap.conf.in
> @@ -16,19 +16,19 @@ source=##DISTRO_APT_SOURCE##
>   suite=##DISTRO_SUITE##
>   components=##DISTRO_COMPONENTS##
>   packages=##IMAGE_PREINSTALL##
> -omitdebsrc=true
> +omitdebsrc=false
>   
>   [updates]
>   source=##DISTRO_APT_SOURCE##
>   suite=##DISTRO_SUITE##-updates
>   components=##DISTRO_COMPONENTS##
> -omitdebsrc=true
> +omitdebsrc=false
>   
>   [security]
>   source=##DISTRO_APT_SOURCE_SEC##
>   suite=##DISTRO_SUITE##/updates
>   components=##DISTRO_COMPONENTS##
> -omitdebsrc=true
> +omitdebsrc=false
>   
>   [Isar]
>   packages=##IMAGE_INSTALL##
> diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
> index dea2cdb..6ce3d6d 100644
> --- a/meta-isar/recipes-core/images/isar-image-base.bb
> +++ b/meta-isar/recipes-core/images/isar-image-base.bb
> @@ -52,6 +52,79 @@ do_rootfs_populate_dev() {
>       _mknod_cdev 666 ptmx    5 2
>   }
>   
> +do_populate_repo() {
> +
> +    APT_CACHEDIR=${IMAGE_ROOTFS}/var/cache/apt # Home of the apt cache (for the target image)
> +    APT_DEBDIR=${APT_CACHEDIR}/archives        # Downloaded packages can be found there
> +    APT_SRCDIR=${WORKDIR}/apt-sources          # Put source packages outside of the rootfs
> +
> +    sudo mkdir -p ${APT_SRCDIR}
> +
> +    apt_status_file=$(mktemp)
> +    for p in ${DEPLOY_DIR_DEB}/${PN}/*.deb; do
> +        # Only process packages that were NOT built by Isar (already added to the repo)
> +        deb=$(basename ${p})
> +        if [ ! -f ${DEPLOY_DIR_DEB}/${deb} ]; then
> +            # Extract package name
> +            pkg=$(dpkg-deb -I ${p}|grep -e '^ Package:'|sed -e 's,^ Package: ,,g')
> +
> +            # Extract package section and priority so we can tell rerepro where to place this package
> +            sect=$(dpkg-deb -I ${p}|grep -e '^ Section:'|sed -e 's,^ Section: ,,g')
> +            prio=$(dpkg-deb -I ${p}|grep -e '^ Priority:'|sed -e 's,^ Priority: ,,g')
> +
> +            # Determine name of the source package (same as the binary package if not specified)
> +            src=$(dpkg-deb -I ${p}|grep -e '^ Source:'|sed -e 's,^ Source: ,,g'|awk '{ print $1; }')
> +            src=${src:-${pkg}}
> +
> +            # Add the binary package to the repo
> +            call_reprepro -b ${DEPLOY_DIR_APT}/${DISTRO}   \
> +                --dbdir ${DEPLOY_DIR_DB}/${DISTRO}         \
> +                -C main -S ${sect} -P ${prio}              \
> +                includedeb ${DEBDISTRONAME} ${p}
> +
> +            # Get source package and add it to the repo
> +            # Notes:
> +            #   - multistrap --source-dir is failing to fetch sources hence this manual process
> +            #     (the culprit appears to be the following apt-get option: APT::Default-Release='*')
> +            #   - the apt-get command-line was copied from multistrap and tweaked for sources
> +            #     (a separate/temporary apt status file is used to avoid polluting the existing one
> +            #      with source transactions)
> +            #   - as we assume sources were already fetched if the directory exists, quietly
> +            #     attempt to delete it first; if sources are indeed there, it will fetch. If
> +            #     they aren't, the directory will be deleted and we will have another go at it
> +            rmdir ${APT_SRCDIR}/${src} 2>/dev/null || true
> +            if [ ! -d "${APT_SRCDIR}/${src}" ]; then
> +                sudo mkdir -p ${APT_SRCDIR}/${src}
> +                cd ${APT_SRCDIR}/${src}
> +                bbnote "fetching source package '${src}' (binary: ${pkg})"
> +                sudo -E apt-get \
> +                    -o Dir::Etc::TrustedParts=${IMAGE_ROOTFS}/etc/apt/trusted.gpg.d \
> +                    -o Dir::Etc::Trusted=${IMAGE_ROOTFS}/etc/apt/trusted.gpg \
> +                    -o Apt::Get::AllowUnauthenticated=true \
> +                    -o Apt::Get::Download-Only=true \
> +                    -o Apt::Install-Recommends=false \
> +                    -o Dir=${IMAGE_ROOTFS} \
> +                    -o Dir::Etc=${IMAGE_ROOTFS}/etc/apt/ \
> +                    -o Dir::Etc::Parts=${IMAGE_ROOTFS}/etc/apt/apt.conf.d/ \
> +                    -o Dir::Etc::PreferencesParts=${IMAGE_ROOTFS}/etc/apt/preferences.d/ \
> +                    -o Dir::State=${IMAGE_ROOTFS}/var/lib/apt/ \
> +                    -o Dir::State::Status=${apt_status_file} \
> +                    -o Dir::Cache=${IMAGE_ROOTFS}/var/cache/apt/ -y source ${src}
> +                call_reprepro -b ${DEPLOY_DIR_APT}/${DISTRO}   \
> +                    --dbdir ${DEPLOY_DIR_DB}/${DISTRO}         \
> +                    -C main -S ${sect} -P ${prio}              \
> +                    includedsc ${DEBDISTRONAME}                \
> +                    ${APT_SRCDIR}/${src}/*.dsc
> +            else
> +                bbnote "already fetched source package '${src}' (binary: ${pkg})"
> +            fi
> +        fi
> +    done
> +    rm -f ${apt_status_file}
> +}
> +
> +addtask populate_repo
> +
>   do_rootfs() {
>       E="${@ bb.utils.export_proxies(d)}"
>   
> @@ -94,6 +167,11 @@ do_rootfs() {
>       # because of proxy settings
>       sudo -E multistrap -a ${DISTRO_ARCH} -d "${IMAGE_ROOTFS}" -f "${WORKDIR}/multistrap.conf"
>   
> +    # Copy packages before they vanish when the root file-systems get configured
> +    # (apt purge is being run)
> +    install -m 755 -d ${DEPLOY_DIR_DEB}/${PN}
> +    sudo cp -l ${IMAGE_ROOTFS}/var/cache/apt/archives/*.deb ${DEPLOY_DIR_DEB}/${PN}/
> +
>       # Configure root filesystem
>       sudo chroot ${IMAGE_ROOTFS} /${DISTRO_CONFIG_SCRIPT} ${MACHINE_SERIAL} ${BAUDRATE_TTY} \
>           ${ROOTFS_DEV}
> 

-- 
With best regards,
Alexander Smirnov

ilbers GmbH
Baierbrunner Str. 28c
D-81379 Munich
+49 (89) 122 67 24-0
http://ilbers.de/
Commercial register Munich, HRB 214197
General manager: Baurzhan Ismagulov