From: Henning Schild <henning.schild@siemens.com>
To: isar-users@googlegroups.com
Cc: Vijai Kumar K <vijaikumar.kanagarajan@gmail.com>,
Henning Schild <henning.schild@siemens.com>
Subject: [PATCHv4 23/26] base-apt: pull base-apt population to the front of the build chain
Date: Fri, 31 Jan 2020 15:29:57 +0100 [thread overview]
Message-ID: <20200131143000.14873-24-henning.schild@siemens.com> (raw)
In-Reply-To: <20200131143000.14873-1-henning.schild@siemens.com>
From: Henning Schild <henning.schild@siemens.com>
Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.
The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass
diff --git a/doc/user_manual.md b/doc/user_manual.md
index c67c1778..ebf57940 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -823,10 +823,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```
- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.
```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```
- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -836,14 +836,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.
```
sudo rm -rf tmp
```
- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):
```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326e..00000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
- # Check if this package is already in base-apt
- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
-
-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b687fcfb..cfd617af 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 557756c7..54b5e5c9 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index d86c5b92..e46142da 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 2078115b..a93c4e3b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():
do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] = "base-apt:do_cache"
isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index c9a7dad1..2dba7794 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
+
+ # Check if this package is already in base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
"${KEYFILES}"
+
+ if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+ [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ # We would need two separate repository paths for that.
+ # Otherwise packages (especially the 'all' arch ones) from one
+ # distribution can influence the package versions of the other
+ # distribution.
+ bbfatal "Different host and target distributions are currently not supported." \
+ "Try it without cross-build."
+ fi
+
+ populate_base_apt
}
-python do_cache_config() {
+python do_cache() {
+ if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ return 0
+
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
filename = fetcher.localpath(key)
d.appendVar("KEYFILES", " %s" % filename)
- bb.build.exec_func('repo_config', d)
+ bb.build.exec_func('repo', d)
}
-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee91..733ba86e 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
# Enable use of unsigned cached base repository
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
--
2.24.1
next prev parent reply other threads:[~2020-01-31 14:30 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-01-31 14:29 [PATCHv4 00/26] base-apt-rework Henning Schild
2020-01-31 14:29 ` [PATCHv4 01/26] repository: new class to deal with repos Henning Schild
2020-01-31 14:29 ` [PATCHv4 02/26] dpkg-base: add download caching of apt:// downloads Henning Schild
2020-01-31 14:29 ` [PATCHv4 03/26] meta: dpkg-base: convert "exit" into "return 0" Henning Schild
2020-01-31 14:29 ` [PATCHv4 04/26] base-apt: change the sources.list to also offer deb-src Henning Schild
2020-01-31 14:29 ` [PATCHv4 05/26] base-apt: add deb-src packages as well Henning Schild
2020-01-31 14:29 ` [PATCHv4 06/26] base-apt: do not skip gpg check when it is signed Henning Schild
2020-01-31 14:29 ` [PATCHv4 07/26] ci: conf: add "hello" to the sample config and every build Henning Schild
2020-01-31 14:29 ` [PATCHv4 08/26] meta: split all apt-get invocations into download and execution Henning Schild
2020-01-31 14:29 ` [PATCHv4 09/26] meta: create DL_DIR support for all apt-get downloaded .debs Henning Schild
2020-01-31 14:29 ` [PATCHv4 10/26] meta: import DL_DIR debs before apt-get download steps Henning Schild
2020-01-31 14:29 ` [PATCHv4 11/26] meta: include DL_DIR deb import/export into bootstrap Henning Schild
2020-01-31 14:29 ` [PATCHv4 12/26] base-apt: populate from DEBDIR as well Henning Schild
2020-01-31 14:29 ` [PATCHv4 13/26] base-apt: drop the "apt_cache" feature Henning Schild
2020-01-31 14:29 ` [PATCHv4 14/26] base-apt: do not copy debs directly out of rootfss anymore Henning Schild
2020-01-31 14:29 ` [PATCHv4 15/26] base-apt: rework base-apt population Henning Schild
2020-01-31 14:29 ` [PATCHv4 16/26] base-apt: move class "base-apt-helper" into only user Henning Schild
2020-01-31 14:29 ` [PATCHv4 17/26] CI: include "isar-disable-apt-cache" into all CI images Henning Schild
2020-01-31 14:29 ` [PATCHv4 18/26] CI: include "cowsay" into default build to test dpkg-gbp Henning Schild
2020-01-31 14:29 ` [PATCHv4 19/26] CI: set BB_NO_NETWORK for cached rebuild Henning Schild
2020-01-31 14:29 ` [PATCHv4 20/26] meta: repository: implement repo_contains_package and use it in base_apt Henning Schild
2020-01-31 14:29 ` [PATCHv4 21/26] meta: repository: simplify the matching of packages Henning Schild
2020-01-31 14:29 ` [PATCHv4 22/26] meta: deb-dl-dir: do not cache debs from isar-apt Henning Schild
2020-02-03 17:20 ` Jan Kiszka
2020-02-03 18:27 ` Henning Schild
2020-02-03 18:59 ` Henning Schild
2020-02-04 2:11 ` Su, Bao Cheng
2020-02-04 5:10 ` Su, Bao Cheng
2020-02-04 5:14 ` Jan Kiszka
2020-02-04 5:31 ` Su, Bao Cheng
2020-02-04 16:26 ` Henning Schild
2020-02-04 9:33 ` Henning Schild
2020-02-05 12:11 ` Henning Schild
2020-02-05 14:21 ` Su, Bao Cheng
2020-01-31 14:29 ` Henning Schild [this message]
2020-01-31 14:29 ` [PATCHv4 24/26] meta: base-apt: remove isar-apt check from population Henning Schild
2020-01-31 14:29 ` [PATCHv4 25/26] meta: deb-dl-dir: only export newly downloaded files Henning Schild
2020-01-31 14:30 ` [PATCHv4 26/26] CI: test a custom kernel build in the base-apt offline rebuild Henning Schild
2020-01-31 14:30 ` [PATCHv4 00/26] base-apt-rework Henning Schild
2020-03-04 23:25 ` Baurzhan Ismagulov
2020-03-04 23:28 ` [PATCH v5 01/26] repository: new class to deal with repos Baurzhan Ismagulov
2020-03-10 13:14 ` Henning Schild
2020-03-10 14:13 ` Baurzhan Ismagulov
2020-03-04 23:28 ` [PATCH v5 08/26] meta: split all apt-get invocations into download and execution Baurzhan Ismagulov
2020-03-10 13:01 ` [PATCHv4 00/26] base-apt-rework Henning Schild
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200131143000.14873-24-henning.schild@siemens.com \
--to=henning.schild@siemens.com \
--cc=isar-users@googlegroups.com \
--cc=vijaikumar.kanagarajan@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox