Hi Henning, Can't we have the base-apt repo generation towards the end of the build chain? Right now if you need to generate a base-apt then you need to build twice. Once without ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO set. What if we could just generate the repo towards the end nevertheless and let user decide whether to use the repo in the subsequent build by setting ISAR_USE_CACHED_BASE_REPO? Thanks, Vijai Kumar K On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning Schild wrote: > > From: Henning Schild > > > Users had to call a special task before switching to using base-apt. > With this commit you flip one switch and the repo will be created as one > of the first steps. > > The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is > "processed" files and not just a download-cache. > > Signed-off-by: Henning Schild > > --- > doc/user_manual.md | 8 +-- > meta/classes/image-cache-extension.bbclass | 64 ------------------ > meta/classes/image.bbclass | 1 - > meta/classes/rootfs.bbclass | 2 +- > meta/conf/bitbake.conf | 4 +- > .../isar-bootstrap/isar-bootstrap.inc | 1 + > meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++--- > scripts/ci_build.sh | 4 +- > 8 files changed, 69 insertions(+), 82 deletions(-) > delete mode 100644 meta/classes/image-cache-extension.bbclass > > diff --git a/doc/user_manual.md b/doc/user_manual.md > index 4d699bd..aaf9638 100644 > --- a/doc/user_manual.md > +++ b/doc/user_manual.md > @@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.: > BASE_REPO_KEY = "file://"' > ``` > > - - Trigger creation of local apt caching Debian packages during image > generation. > + - Trigger the download and caching of all required files by doing a > warm-up build. > > ``` > -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base > +bitbake mc:qemuarm-buster:isar-image-base > ``` > > - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`: > @@ -835,14 +835,14 @@ bitbake -c cache_base_repo > mc:qemuarm-buster:isar-image-base > #ISAR_USE_CACHED_BASE_REPO ?= "1" > #BB_NO_NETWORK ?= "1" > ``` > - - Remove build artifacts to use only local base-apt: > + - Remove build artifacts to use only local base-apt, in fact toggling > ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is > just the way to be extra sure that only the download cache is used. > > ``` > sudo rm -rf tmp > > ``` > > - - Trigger again generation of image (now using local caching repo): > + - Trigger the generation of your image again (now a local repo will be > created out of the download cache from the last run): > > ``` > bitbake mc:qemuarm-buster:isar-image-base > diff --git a/meta/classes/image-cache-extension.bbclass > b/meta/classes/image-cache-extension.bbclass > deleted file mode 100644 > index 4123326..0000000 > --- a/meta/classes/image-cache-extension.bbclass > +++ /dev/null > @@ -1,64 +0,0 @@ > -# This software is a part of ISAR. > -# Copyright (C) Siemens AG, 2019 > -# > -# SPDX-License-Identifier: MIT > -# > -# This class extends the image.bbclass to supply the creation of cache > repositories > - > -inherit repository > - > -populate_base_apt() { > - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do > - # NOTE: due to packages stored by reprepro are not modified, we > can > - # use search by filename to check if package is already in repo. > In > - # addition, md5sums are compared to ensure that the package is > the > - # same and should not be overwritten. This method is easier and > more > - # robust than querying reprepro by name. > - > - # Check if this package is taken from Isar-apt, if so - ingore > it. > - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" > && \ > - continue > - > - # Check if this package is already in base-apt > - ret=0 > - repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" > "${package}" || > - ret=$? > - [ "${ret}" = "0" ] && continue > - if [ "${ret}" = "1" ]; then > - repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${base_apt_p}" > - fi > - > - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${package}" > - done > - > - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; > do > - repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > - "${BASE_DISTRO_CODENAME}" \ > - "${package}" > - done > -} > - > -do_cache_base_repo[depends] = "base-apt:do_cache_config" > -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}" > -do_cache_base_repo() { > - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] && > - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then > - # We would need two separate repository paths for that. > - # Otherwise packages (especially the 'all' arch ones) from one > - # distribution can influence the package versions of the other > - # distribution. > - bbfatal "Different host and target distributions are currently > not supported." \ > - "Try it without cross-build." > - fi > - > - populate_base_apt > -} > -addtask cache_base_repo after do_rootfs do_install_imager_deps > diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass > index b687fcf..cfd617a 100644 > --- a/meta/classes/image.bbclass > +++ b/meta/classes/image.bbclass > @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}" > > inherit rootfs > inherit image-sdk-extension > -inherit image-cache-extension > inherit image-tools-extension > inherit image-postproc-extension > inherit image-locales-extension > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > index 6d4d42c..a10e7cc 100644 > --- a/meta/classes/rootfs.bbclass > +++ b/meta/classes/rootfs.bbclass > @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() { > > do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}" > do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} > ${ROOTFS_INSTALL_COMMAND}" > -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build > isar-apt:do_cache_config" > +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build > isar-apt:do_cache_config base-apt:do_cache" > do_rootfs_install[deptask] = "do_deploy_deb" > python do_rootfs_install() { > configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or > "").split() > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index d86c5b9..e46142d 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db" > THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg" > > # Base apt repository paths > -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt" > -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db" > +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt" > +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db" > > BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH > BBSERVER DL_DIR \ > SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL > TERM \ > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > index 5b14cd3..57cdea9 100644 > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc > @@ -219,6 +219,7 @@ def get_host_release(): > > do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS" > do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}" > +do_bootstrap[depends] = "base-apt:do_cache" > > isar_bootstrap() { > deb_dl_dir_import "${ROOTFSDIR}" > diff --git a/meta/recipes-devtools/base-apt/base-apt.bb > b/meta/recipes-devtools/base-apt/base-apt.bb > index c9a7dad..2dba779 100644 > --- a/meta/recipes-devtools/base-apt/base-apt.bb > +++ b/meta/recipes-devtools/base-apt/base-apt.bb > @@ -9,26 +9,77 @@ inherit repository > BASE_REPO_KEY ?= "" > KEYFILES ?= "" > > -do_cache_config[stamp-extra-info] = "${DISTRO}" > -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > +populate_base_apt() { > + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do > + # NOTE: due to packages stored by reprepro are not modified, we > can > + # use search by filename to check if package is already in repo. > In > + # addition, md5sums are compared to ensure that the package is > the > + # same and should not be overwritten. This method is easier and > more > + # robust than querying reprepro by name. > > -# Generate reprepro config for current distro if it doesn't exist. Once > it's > -# generated, this task should do nothing. > -repo_config() { > + # Check if this package is taken from Isar-apt, if so - ingore > it. > + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" > && \ > + continue > + > + # Check if this package is already in base-apt > + ret=0 > + repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" > "${package}" || > + ret=$? > + [ "${ret}" = "0" ] && continue > + if [ "${ret}" = "1" ]; then > + repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${base_apt_p}" > + fi > + > + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${package}" > + done > + > + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; > do > + repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > + "${BASE_DISTRO_CODENAME}" \ > + "${package}" > + done > +} > + > +do_cache[stamp-extra-info] = "${DISTRO}" > +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock" > + > +repo() { > repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \ > "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \ > "${BASE_DISTRO_CODENAME}" \ > "${KEYFILES}" > + > + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] && > + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then > + # We would need two separate repository paths for that. > + # Otherwise packages (especially the 'all' arch ones) from one > + # distribution can influence the package versions of the other > + # distribution. > + bbfatal "Different host and target distributions are currently > not supported." \ > + "Try it without cross-build." > + fi > + > + populate_base_apt > } > > -python do_cache_config() { > +python do_cache() { > + if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')): > + return 0 > + > for key in d.getVar('BASE_REPO_KEY').split(): > d.appendVar("SRC_URI", " %s" % key) > fetcher = bb.fetch2.Fetch([key], d) > filename = fetcher.localpath(key) > d.appendVar("KEYFILES", " %s" % filename) > > - bb.build.exec_func('repo_config', d) > + bb.build.exec_func('repo', d) > } > > -addtask cache_config after do_unpack before do_build > +addtask cache after do_unpack before do_build > diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh > index 7da3ee9..733ba86 100755 > --- a/scripts/ci_build.sh > +++ b/scripts/ci_build.sh > @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then > > # Enable use of signed cached base repository > echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> > conf/local.conf > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED > + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED > while [ -e bitbake.sock ]; do sleep 1; done > sudo rm -rf tmp > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf > @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then > sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf > > # Enable use of unsigned cached base repository > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET > + bitbake $BB_ARGS $REPRO_TARGETS_SET > while [ -e bitbake.sock ]; do sleep 1; done > sudo rm -rf tmp > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf > -- > 2.24.1 > >