From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6771003121820762112 Date: Tue, 21 Jan 2020 00:55:43 -0800 (PST) From: vijai kumar To: isar-users Message-Id: In-Reply-To: <20191216115011.17664-24-henning.schild@siemens.com> References: <20191216115011.17664-1-henning.schild@siemens.com> <20191216115011.17664-24-henning.schild@siemens.com> Subject: Re: [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_1657_1981687718.1579596943170" X-Google-Token: EI_5mvEFZ4ztVEk5N_A0 X-Google-IP: 139.181.36.34 X-TUID: MMwPdXYQncal ------=_Part_1657_1981687718.1579596943170 Content-Type: multipart/alternative; boundary="----=_Part_1658_1986096349.1579596943171" ------=_Part_1658_1986096349.1579596943171 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit Hi Henning, Can't we have the base-apt repo generation towards the end of the build chain? Right now if you need to generate a base-apt then you need to build twice. Once without ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO set. What if we could just generate the repo towards the end nevertheless and let user decide whether to use the repo in the subsequent build by setting ISAR_USE_CACHED_BASE_REPO? Thanks, Vijai Kumar K On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning Schild wrote: > > From: Henning Schild > > > Users had to call a special task before switching to using base-apt. > With this commit you flip one switch and the repo will be created as one > of the first steps. > > The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is > "processed" files and not just a download-cache. > > Signed-off-by: Henning Schild > > --- > doc/user_manual.md | 8 +-- > meta/classes/image-cache-extension.bbclass | 64 ------------------ > meta/classes/image.bbclass | 1 - > meta/classes/rootfs.bbclass | 2 +- > meta/conf/bitbake.conf | 4 +- > .../isar-bootstrap/isar-bootstrap.inc | 1 + > meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++--- > scripts/ci_build.sh | 4 +- > 8 files changed, 69 insertions(+), 82 deletions(-) > delete mode 100644 meta/classes/image-cache-extension.bbclass > > diff --git a/doc/user_manual.md b/doc/user_manual.md > index 4d699bd..aaf9638 100644 > --- a/doc/user_manual.md > +++ b/doc/user_manual.md > @@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.: > BASE_REPO_KEY = "file://"' > ``` > > - - Trigger creation of local apt caching Debian packages during image > generation. > + - Trigger the download and caching of all required files by doing a > warm-up build. > > ``` > -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base > +bitbake mc:qemuarm-buster:isar-image-base > ``` > > - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`: > @@ -835,14 +835,14 @@ bitbake -c cache_base_repo > mc:qemuarm-buster:isar-image-base > #ISAR_USE_CACHED_BASE_REPO ?= "1" > #BB_NO_NETWORK ?= "1" > ``` > - - Remove build artifacts to use only local base-apt: > + - Remove build artifacts to use only local base-apt, in fact toggling > ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is > just the way to be extra sure that only the download cache is used. > > ``` > sudo rm -rf tmp > > ``` > > - - Trigger again generation of image (now using local caching repo): > + - Trigger the generation of your image again (now a local repo will be > created out of the download cache from the last run): > > ``` > bitbake mc:qemuarm-buster:isar-image-base > diff --git a/meta/classes/image-cache-extension.bbclass > b/meta/classes/image-cache-extension.bbclass > deleted file mode 100644 > index 4123326..0000000 > --- a/meta/classes/image-cache-extension.bbclass > +++ /dev/null > @@ -1,64 +0,0 @@ > -# This software is a part of ISAR. > -# Copyright (C) Siemens AG, 2019 > -# > -# SPDX-License-Identifier: MIT > -# > -# This class extends the image.bbclass to supply the creation of cache > repositories > - > -inherit repository > - > -populate_base_apt() { > - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do > - # NOTE: due to packages stored by reprepro are not modified, we > can > - # use search by filename to check if package is already in repo. > In > - # addition, md5sums are compared to ensure that the package is > the > - # same and should not be overwritten. This method is easier and > more > - # robust than querying reprepro by name. > - > - # Check if this package is taken from Isar-apt, if so - ingore > it. > - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" > && \ > - continue > - > - # Check if this package is already in base-apt > - ret=0 > - repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" > "${package}" || > - ret=$? > - [ "${ret}" = "0" ] && continue > - if [ "${ret}" = "1" ]; then > - repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${base_apt_p}" > - fi > - > - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${package}" > - done > - > - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; > do > - repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${package}" > - done > -} > - > -do_cache_base_repo[depends] = "base-apt:do_cache_config" > -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}" > -do_cache_base_repo() { > - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] && > - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then > - # We would need two separate repository paths for that. > - # Otherwise packages (especially the 'all' arch ones) from one > - # distribution can influence the package versions of the other > - # distribution. > - bbfatal "Different host and target distributions are currently > not supported." \ > - "Try it without cross-build." > - fi > - > - populate_base_apt > -} > -addtask cache_base_repo after do_rootfs do_install_imager_deps > diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass > index b687fcf..cfd617a 100644 > --- a/meta/classes/image.bbclass > +++ b/meta/classes/image.bbclass > @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}" > > inherit rootfs > inherit image-sdk-extension > -inherit image-cache-extension > inherit image-tools-extension > inherit image-postproc-extension > inherit image-locales-extension > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > index 6d4d42c..a10e7cc 100644 > --- a/meta/classes/rootfs.bbclass > +++ b/meta/classes/rootfs.bbclass > @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() { > > do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}" > do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} > ${ROOTFS_INSTALL_COMMAND}" > -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build > isar-apt:do_cache_config" > +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build > isar-apt:do_cache_config base-apt:do_cache" > do_rootfs_install[deptask] = "do_deploy_deb" > python do_rootfs_install() { > configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or > "").split() > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index d86c5b9..e46142d 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db" > THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg" > > # Base apt repository paths > -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt" > -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db" > +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt" > +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db" > > BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH > BBSERVER DL_DIR \ > SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL > TERM \ > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > index 5b14cd3..57cdea9 100644 > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > @@ -219,6 +219,7 @@ def get_host_release(): > > do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS" > do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}" > +do_bootstrap[depends] = "base-apt:do_cache" > > isar_bootstrap() { > deb_dl_dir_import "${ROOTFSDIR}" > diff --git a/meta/recipes-devtools/base-apt/base-apt.bb > b/meta/recipes-devtools/base-apt/base-apt.bb > index c9a7dad..2dba779 100644 > --- a/meta/recipes-devtools/base-apt/base-apt.bb > +++ b/meta/recipes-devtools/base-apt/base-apt.bb > @@ -9,26 +9,77 @@ inherit repository > BASE_REPO_KEY ?= "" > KEYFILES ?= "" > > -do_cache_config[stamp-extra-info] = "${DISTRO}" > -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > +populate_base_apt() { > + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do > + # NOTE: due to packages stored by reprepro are not modified, we > can > + # use search by filename to check if package is already in repo. > In > + # addition, md5sums are compared to ensure that the package is > the > + # same and should not be overwritten. This method is easier and > more > + # robust than querying reprepro by name. > > -# Generate reprepro config for current distro if it doesn't exist. Once > it's > -# generated, this task should do nothing. > -repo_config() { > + # Check if this package is taken from Isar-apt, if so - ingore > it. > + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" > && \ > + continue > + > + # Check if this package is already in base-apt > + ret=0 > + repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" > "${package}" || > + ret=$? > + [ "${ret}" = "0" ] && continue > + if [ "${ret}" = "1" ]; then > + repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${base_apt_p}" > + fi > + > + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${package}" > + done > + > + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; > do > + repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${package}" > + done > +} > + > +do_cache[stamp-extra-info] = "${DISTRO}" > +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > + > +repo() { > repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > "${BASE_DISTRO_CODENAME}" \ > "${KEYFILES}" > + > + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] && > + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then > + # We would need two separate repository paths for that. > + # Otherwise packages (especially the 'all' arch ones) from one > + # distribution can influence the package versions of the other > + # distribution. > + bbfatal "Different host and target distributions are currently > not supported." \ > + "Try it without cross-build." > + fi > + > + populate_base_apt > } > > -python do_cache_config() { > +python do_cache() { > + if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): > + return 0 > + > for key in d.getVar('BASE_REPO_KEY').split(): > d.appendVar("SRC_URI", " %s" % key) > fetcher = bb.fetch2.Fetch([key], d) > filename = fetcher.localpath(key) > d.appendVar("KEYFILES", " %s" % filename) > > - bb.build.exec_func('repo_config', d) > + bb.build.exec_func('repo', d) > } > > -addtask cache_config after do_unpack before do_build > +addtask cache after do_unpack before do_build > diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh > index 7da3ee9..733ba86 100755 > --- a/scripts/ci_build.sh > +++ b/scripts/ci_build.sh > @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then > > # Enable use of signed cached base repository > echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> > conf/local.conf > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED > + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED > while [ -e bitbake.sock ]; do sleep 1; done > sudo rm -rf tmp > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf > @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then > sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf > > # Enable use of unsigned cached base repository > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET > + bitbake $BB_ARGS $REPRO_TARGETS_SET > while [ -e bitbake.sock ]; do sleep 1; done > sudo rm -rf tmp > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf > -- > 2.24.1 > > ------=_Part_1658_1986096349.1579596943171 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable
Hi Henning,

Can't we hav= e the base-apt repo generation towards the end of the build chain?

Right now if you need to generate a base-apt then you need= to build twice. Once without
ISAR_USE_CACHED_BASE_REPO and once = with ISAR_USE_CACHED_BASE_REPO set.

What if we cou= ld just generate the repo towards the end nevertheless and let user decide<= /div>
whether to use the repo in the subsequent build by setting ISAR_U= SE_CACHED_BASE_REPO?

Thanks,
Vijai Kumar= K


On Monday, December 16, 2019 at 5:20:16 PM U= TC+5:30, Henning Schild wrote:
= From: Henning Schild <hennin...@siemens.com>

Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as on= e
of the first steps.

The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.

Signed-off-by: Henning Schild <hennin...@siemens.com>
---
=C2=A0doc/user_manual.md =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| =C2=A08 +--
=C2=A0meta/classes/image-cache-extension.bbclass =C2=A0 =C2=A0| 64= ------------------
=C2=A0meta/classes/image.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| =C2=A01 -
=C2=A0meta/classes/rootfs.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 | =C2=A02 +-
=C2=A0meta/conf/bitbake.conf =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| =C2=A04 +-
=C2=A0.../isar-bootstrap/isar-bootstrap.inc =C2=A0 =C2=A0 =C2=A0 = =C2=A0 | =C2=A01 +
=C2=A0meta/recipes-devtools/base-apt/base-apt.bb =C2=A0 =C2=A0| 67 ++++++++++++++++---
=C2=A0scripts/ci_build.sh =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 | =C2=A04 +-
=C2=A08 files changed, 69 insertions(+), 82 deletions(-)
=C2=A0delete mode 100644 meta/classes/image-cache-extension.bbclas= s

diff --git a/doc/user_manual.md b/doc/user_manual.md
index 4d699bd..aaf9638 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.= :
=C2=A0BASE_REPO_KEY =3D "file://<absolute_path_to_your_pub= _key_file>"'
=C2=A0```
=C2=A0
- - Trigger creation of local apt caching Debian packages during image = generation.
+ - Trigger the download and caching of all required files by doing a w= arm-up build.
=C2=A0
=C2=A0```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
=C2=A0```
=C2=A0
=C2=A0 - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -835,14 +835,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar= -image-base
=C2=A0#ISAR_USE_CACHED_BASE_REPO ?=3D "1"
=C2=A0#BB_NO_NETWORK ?=3D "1"
=C2=A0```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling= ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is j= ust the way to be extra sure that only the download cache is used.
=C2=A0
=C2=A0```
=C2=A0sudo rm -rf tmp
=C2=A0
=C2=A0```
=C2=A0
- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will b= e created out of the download cache from the last run):
=C2=A0
=C2=A0```
=C2=A0bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/cla= sses/image-cache-extension.bbclass
deleted file mode 100644
index 4123326..0000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache= repositories
-
-inherit repository
-
-populate_base_apt() {
- =C2=A0 =C2=A0find "${DEBDIR}"/"${DISTRO}" -name &= #39;*\.deb' | while read package; do
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# NOTE: due to packages stored by reprepro= are not modified, we can
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# use search by filename to check if packa= ge is already in repo. In
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# addition, md5sums are compared to ensure= that the package is the
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# same and should not be overwritten. This= method is easier and more
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# robust than querying reprepro by name.
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# Check if this package is taken from Isar= -apt, if so - ingore it.
- =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_contains_package "${REPO_ISAR_DI= R}/${DISTRO}" "${package}" && \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0continue
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# Check if this package is already in base= -apt
- =C2=A0 =C2=A0 =C2=A0 =C2=A0ret=3D0
-=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0repo_contains_package = "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||= =20
-=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0 =C2=A0 =C2=A0ret=3D$?
- =C2=A0 =C2=A0 =C2=A0 =C2=A0[ "${ret}" =3D "0" ] &= amp;& continue
- =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ "${ret}" =3D "1" = ]; then
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_del_package "${REP= O_BASE_DIR}"/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_B= ASE_DB_DIR}"/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_D= ISTRO_CODENAME}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${base_a= pt_p}"
- =C2=A0 =C2=A0 =C2=A0 =C2=A0fi
-
- =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_add_packages "${REPO_BASE_DIR}&q= uot;/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_BASE_DB_DIR}&qu= ot;/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_DISTRO_CODENAME= }" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${package}"
- =C2=A0 =C2=A0done
-
- =C2=A0 =C2=A0find "${DEBSRCDIR}"/"${DISTRO}" -nam= e '*\.dsc' | while read package; do
- =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_add_srcpackage "${REPO_BASE_DIR}= "/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_BASE_DB_DIR}&qu= ot;/"${BASE_DISTRO}" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_DISTRO_CODENAME= }" \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${package}"
- =C2=A0 =C2=A0done
-}
-
-do_cache_base_repo[depends] =3D "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] =3D "${REPO_BASE_DIR}/isar.lock&quo= t;
-do_cache_base_repo[stamp-extra-info] =3D "${DISTRO}-${MACHIN= E}"
-do_cache_base_repo() {
- =C2=A0 =C2=A0if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt&= #39; ] &&
- =C2=A0 =C2=A0 =C2=A0 =C2=A0[ '${DISTRO}' !=3D '${HOST_DIS= TRO}' ]; then
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# We would need two separate repository pa= ths for that.
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# Otherwise packages (especially the '= all' arch ones) from one
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# distribution can influence the package v= ersions of the other
- =C2=A0 =C2=A0 =C2=A0 =C2=A0# distribution.
- =C2=A0 =C2=A0 =C2=A0 =C2=A0bbfatal "Different host and target di= stributions are currently not supported." \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"Try it w= ithout cross-build."
- =C2=A0 =C2=A0fi
-
- =C2=A0 =C2=A0populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b687fcf..cfd617a 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?=3D "${DEPLOY_DIR_IM= AGE}"
=C2=A0
=C2=A0inherit rootfs
=C2=A0inherit image-sdk-extension
-inherit image-cache-extension
=C2=A0inherit image-tools-extension
=C2=A0inherit image-postproc-extension
=C2=A0inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 6d4d42c..a10e7cc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
=C2=A0
=C2=A0do_rootfs_install[root_cleandirs] =3D "${ROOTFSDIR}&quo= t;
=C2=A0do_rootfs_install[vardeps] =3D "${ROOTFS_CONFIGURE_COMMAND} = ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] =3D "isar-bootstrap-${@'target'= ; if d.getVar('ROOTFS_ARCH') =3D=3D d.getVar('DISTRO_ARCH')= else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] =3D "isar-bootstrap-${@'target'= ; if d.getVar('ROOTFS_ARCH') =3D=3D d.getVar('DISTRO_ARCH')= else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache&q= uot;
=C2=A0do_rootfs_install[deptask] =3D "do_deploy_deb"
=C2=A0python do_rootfs_install() {
=C2=A0 =C2=A0 =C2=A0configure_cmds =3D (d.getVar("ROOTFS_CONFIGURE= _COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index d86c5b9..e46142d 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR =3D "${DEPLOY_DIR}/isar-apt/db&= quot;
=C2=A0THIRD_PARTY_APT_KEYRING =3D "/etc/apt/trusted.gpg.d/third_party.gpg"
=C2=A0
=C2=A0# Base apt repository paths
-REPO_BASE_DIR =3D "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR =3D "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR =3D "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"= ;
+REPO_BASE_DB_DIR =3D "${DEPLOY_DIR}/base-apt/${DISTRO}/db&qu= ot;
=C2=A0
=C2=A0BB_HASHBASE_WHITELIST ?=3D "TMPDIR FILE PATH PWD BB_TASKHASH= BBPATH BBSERVER DL_DIR \
=C2=A0 =C2=A0 =C2=A0SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOM= E LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b= /meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5b14cd3..57cdea9 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():
=C2=A0
=C2=A0do_bootstrap[vardeps] +=3D "DISTRO_APT_PREMIRRORS"
=C2=A0do_bootstrap[dirs] =3D "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] =3D "base-apt:do_cache"
=C2=A0
=C2=A0isar_bootstrap() {
=C2=A0 =C2=A0 =C2=A0deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index c9a7dad..2dba779 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
=C2=A0BASE_REPO_KEY ?=3D ""
=C2=A0KEYFILES ?=3D ""
=C2=A0
-do_cache_config[stamp-extra-info] =3D "${DISTRO}"
-do_cache_config[lockfiles] =3D "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ =C2=A0 =C2=A0find "${DEBDIR}"/"${DISTRO}" -name &= #39;*\.deb' | while read package; do
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# NOTE: due to packages stored by reprepro= are not modified, we can
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# use search by filename to check if packa= ge is already in repo. In
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# addition, md5sums are compared to ensure= that the package is the
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# same and should not be overwritten. This= method is easier and more
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# robust than querying reprepro by name.
=C2=A0
-# Generate reprepro config for current distro if it doesn't exist.= Once it's
-# generated, this task should do nothing.
-repo_config() {
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# Check if this package is taken from Isar= -apt, if so - ingore it.
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_contains_package "${REPO_ISAR_DI= R}/${DISTRO}" "${package}" && \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0continue
+
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# Check if this package is already in base= -apt
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0ret=3D0
+=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0repo_contains_package = "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0=C2=A0 =C2=A0 =C2=A0ret=3D$?
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0[ "${ret}" =3D "0" ] &= amp;& continue
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ "${ret}" =3D "1" = ]; then
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_del_package "${REP= O_BASE_DIR}"/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_B= ASE_DB_DIR}"/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_D= ISTRO_CODENAME}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${base_a= pt_p}"
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0fi
+
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_add_packages "${REPO_BASE_DIR}&q= uot;/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_BASE_DB_DIR}&qu= ot;/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_DISTRO_CODENAME= }" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${package}"
+ =C2=A0 =C2=A0done
+
+ =C2=A0 =C2=A0find "${DEBSRCDIR}"/"${DISTRO}" -nam= e '*\.dsc' | while read package; do
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0repo_add_srcpackage "${REPO_BASE_DIR}= "/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_BASE_DB_DIR}&qu= ot;/"${BASE_DISTRO}" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_DISTRO_CODENAME= }" \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${package}"
+ =C2=A0 =C2=A0done
+}
+
+do_cache[stamp-extra-info] =3D "${DISTRO}"
+do_cache[lockfiles] =3D "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
=C2=A0 =C2=A0 =C2=A0repo_create "${REPO_BASE_DIR}"/"${BA= SE_DISTRO}" \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${REPO_BASE_DB_DIR}"/"= ;${BASE_DISTRO}" \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${BASE_DISTRO_CODENAME}" \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"${KEYFILES}"
+
+ =C2=A0 =C2=A0if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt&= #39; ] &&
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0[ '${DISTRO}' !=3D '${HOST_DIS= TRO}' ]; then
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# We would need two separate repository pa= ths for that.
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# Otherwise packages (especially the '= all' arch ones) from one
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# distribution can influence the package v= ersions of the other
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0# distribution.
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0bbfatal "Different host and target di= stributions are currently not supported." \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"Try it w= ithout cross-build."
+ =C2=A0 =C2=A0fi
+
+ =C2=A0 =C2=A0populate_base_apt
=C2=A0}
=C2=A0
-python do_cache_config() {
+python do_cache() {
+ =C2=A0 =C2=A0if not bb.utils.to_boolean(d.getVar('ISAR_USE_C= ACHED_BASE_REPO')):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0return 0
+
=C2=A0 =C2=A0 =C2=A0for key in d.getVar('BASE_REPO_KEY').s= plit():
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0d.appendVar("SRC_URI", &quo= t; %s" % key)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0fetcher =3D bb.fetch2.Fetch([key], d)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0filename =3D fetcher.localpath(key)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0d.appendVar("KEYFILES", &qu= ot; %s" % filename)
=C2=A0
- =C2=A0 =C2=A0bb.build.exec_func('repo_config', d)
+ =C2=A0 =C2=A0bb.build.exec_func('repo', d)
=C2=A0}
=C2=A0
-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee9..733ba86 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
=C2=A0
=C2=A0 =C2=A0 =C2=A0# Enable use of signed cached base repository
=C2=A0 =C2=A0 =C2=A0echo BASE_REPO_KEY=3D\"file://$ISAR_TESTS= UITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- =C2=A0 =C2=A0bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_S= IGNED
+ =C2=A0 =C2=A0bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
=C2=A0 =C2=A0 =C2=A0while [ -e bitbake.sock ]; do sleep 1; done
=C2=A0 =C2=A0 =C2=A0sudo rm -rf tmp
=C2=A0 =C2=A0 =C2=A0sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=3D &q= uot;1"/ISAR_USE_CACHED_BASE_REPO ?=3D "1"/g' conf/local.= conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
=C2=A0 =C2=A0 =C2=A0sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY= /g' conf/local.conf
=C2=A0
=C2=A0 =C2=A0 =C2=A0# Enable use of unsigned cached base repository
- =C2=A0 =C2=A0bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ =C2=A0 =C2=A0bitbake $BB_ARGS $REPRO_TARGETS_SET
=C2=A0 =C2=A0 =C2=A0while [ -e bitbake.sock ]; do sleep 1; done
=C2=A0 =C2=A0 =C2=A0sudo rm -rf tmp
=C2=A0 =C2=A0 =C2=A0sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=3D &q= uot;1"/ISAR_USE_CACHED_BASE_REPO ?=3D "1"/g' conf/local.= conf
--=20
2.24.1

------=_Part_1658_1986096349.1579596943171-- ------=_Part_1657_1981687718.1579596943170--