Re: [PATCH 09/17] meta: import DL_DIR debs before apt-get download steps

[Henning Schild <henning.schild@siemens.com>]

A future patch will actually populate base-apt out of the files in
DL_DIR, i did not sent that yet.

Once the creation+population is done purely from the download cache, we
can not exclude packages from isar-apt anymore.

At the moment i think that would not be a problem because the rebuild
from base-apt will just work. isar-apt will always win and just rebuild
the "same" stuff.

The only problem i see is that base-apt will potentially contain
product packages, which means you can not share it as widely as you can
a pure "mirror", which base-apt is not anyways.

Henning

On Mon, 28 Oct 2019 18:01:24 +0100
Henning Schild <henning.schild@siemens.com> wrote:

> From: Henning Schild <henning.schild@siemens.com>
> 
> If we ever downloaded a file there is no need to do that again. We can
> not only use that DL_DIR to later construct a repository but also to
> speed up our rebuilds.
> 
> Signed-off-by: Henning Schild <henning.schild@siemens.com>
> ---
>  meta/classes/deb-dl-dir.bbclass            | 10 ++++++++++
>  meta/classes/dpkg-gbp.bbclass              |  1 +
>  meta/classes/dpkg.bbclass                  |  1 +
>  meta/classes/image-tools-extension.bbclass |  1 +
>  meta/classes/rootfs.bbclass                |  6 ++++++
>  5 files changed, 19 insertions(+)
> 
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index e58df7a..b137eb9 100644
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -3,6 +3,16 @@
>  #
>  # SPDX-License-Identifier: MIT
>  
> +deb_dl_dir_import() {
> +    export pc="${DEBDIR}/${DISTRO}"
> +    export rootfs="${1}"
> +    [ ! -d "${pc}" ] && exit 0
> +    flock -s "${pc}".lock -c ' \
> +        sudo find "${pc}" -type f -iname '*\.deb' -exec \
> +            cp -f '{}' "${rootfs}"/var/cache/apt/archives/ \;
> +    '
> +}
> +
>  deb_dl_dir_export() {
>      export pc="${DEBDIR}/${DISTRO}"
>      export rootfs="${1}"
> diff --git a/meta/classes/dpkg-gbp.bbclass
> b/meta/classes/dpkg-gbp.bbclass index 9e6945b..be0769c 100644
> --- a/meta/classes/dpkg-gbp.bbclass
> +++ b/meta/classes/dpkg-gbp.bbclass
> @@ -13,6 +13,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
>  do_install_builddeps_append() {
>      dpkg_do_mounts
>      E="${@ bb.build.exec_func('isar_export_proxies', d)}"
> +    deb_dl_dir_import "${BUILDCHROOT_DIR}"
>      sudo -E chroot ${BUILDCHROOT_DIR} \
>          apt-get install -y -o Debug::pkgProblemResolver=yes \
>                          --no-install-recommends --download-only
> ${GBP_DEPENDS} diff --git a/meta/classes/dpkg.bbclass
> b/meta/classes/dpkg.bbclass index b6844e9..9e10ad8 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -7,6 +7,7 @@ inherit dpkg-base
>  do_install_builddeps() {
>      dpkg_do_mounts
>      E="${@ bb.build.exec_func('isar_export_proxies', d)}"
> +    deb_dl_dir_import "${BUILDCHROOT_DIR}"
>      sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
>          ${PP}/${PPS} ${DISTRO_ARCH} --download-only
>      deb_dl_dir_export "${BUILDCHROOT_DIR}"
> diff --git a/meta/classes/image-tools-extension.bbclass
> b/meta/classes/image-tools-extension.bbclass index 3906585..08b426d
> 100644 --- a/meta/classes/image-tools-extension.bbclass
> +++ b/meta/classes/image-tools-extension.bbclass
> @@ -25,6 +25,7 @@ do_install_imager_deps() {
>      buildchroot_do_mounts
>  
>      E="${@ bb.build.exec_func('isar_export_proxies', d)}"
> +    deb_dl_dir_import ${BUILDCHROOT_DIR}
>      sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
>          apt-get update \
>              -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 841aa89..845304c 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
>      fi
>  }
>  
> +ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
> +rootfs_import_package_cache[weight] = "5"
> +rootfs_import_package_cache() {
> +    deb_dl_dir_import ${ROOTFSDIR}
> +}
> +
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>  rootfs_install_pkgs_download[weight] = "600"
>  rootfs_install_pkgs_download[isar-apt-lock] = "release-after"