From: Uladzimir Bely <ubely@ilbers.de>
To: isar-users@googlegroups.com
Subject: [PATCH v4 1/3] Populate base-apt from both DISTRO and HOST_DISTRO download dirs
Date: Mon, 28 Nov 2022 12:05:32 +0100 [thread overview]
Message-ID: <20221128110534.31450-2-ubely@ilbers.de> (raw)
In-Reply-To: <20221128110534.31450-1-ubely@ilbers.de>
Downstreams may want to change DISTRO to some custom value.
When cross-building, this leads to downloading packages for host
and target distros to the different download subdirs.
While base-apt is populated only from DISTRO download subdir,
second cached (ISAR_USE_CACHED_BASE_REPO = "1") build fails due to
missing packages in base-apt.
The similar issue happens with cross-building raspberry targets
since they use their own mirrors and DISTRO.
Fix the issue by splitting base-apt by introducing HOST_BASE_DISTRO
variable that is used instead of BASE_DISTRO for caching builds
for host-related components (e.g., isar-bootstrap, buildchroot-host,
sbuild-chroot-host). Target components still use BASE_DISTRO.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta-isar/conf/distro/raspbian-stretch.conf | 2 +
meta-isar/conf/distro/raspios-bullseye.conf | 2 +
meta-isar/conf/distro/ubuntu-focal.conf | 2 +
meta/conf/distro/debian-common.conf | 2 +
.../isar-bootstrap/isar-bootstrap.inc | 9 +++-
meta/recipes-devtools/base-apt/base-apt.bb | 47 ++++++++++---------
6 files changed, 40 insertions(+), 24 deletions(-)
diff --git a/meta-isar/conf/distro/raspbian-stretch.conf b/meta-isar/conf/distro/raspbian-stretch.conf
index c8f523b3..1ad0152f 100644
--- a/meta-isar/conf/distro/raspbian-stretch.conf
+++ b/meta-isar/conf/distro/raspbian-stretch.conf
@@ -6,6 +6,8 @@
BASE_DISTRO = "raspbian"
BASE_DISTRO_CODENAME = "stretch"
+HOST_BASE_DISTRO = "debian"
+
HOST_DISTRO ?= "debian-stretch"
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
diff --git a/meta-isar/conf/distro/raspios-bullseye.conf b/meta-isar/conf/distro/raspios-bullseye.conf
index 5e4a09c9..1f65c3cd 100644
--- a/meta-isar/conf/distro/raspios-bullseye.conf
+++ b/meta-isar/conf/distro/raspios-bullseye.conf
@@ -6,6 +6,8 @@
BASE_DISTRO = "raspios"
BASE_DISTRO_arm64 = "debian"
+HOST_BASE_DISTRO ?= "debian"
+
BASE_DISTRO_CODENAME = "bullseye"
HOST_DISTRO ?= "debian-bullseye"
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
index 379b5b30..084f8e3d 100644
--- a/meta-isar/conf/distro/ubuntu-focal.conf
+++ b/meta-isar/conf/distro/ubuntu-focal.conf
@@ -8,6 +8,8 @@ require conf/distro/debian-common.conf
BASE_DISTRO = "ubuntu"
BASE_DISTRO_CODENAME = "focal"
+HOST_BASE_DISTRO = "${BASE_DISTRO}"
+
DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}-ports.list"
HOST_DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${HOST_DISTRO}.list conf/distro/${HOST_DISTRO}-ports.list"
diff --git a/meta/conf/distro/debian-common.conf b/meta/conf/distro/debian-common.conf
index c18eebc2..ada7dc6e 100644
--- a/meta/conf/distro/debian-common.conf
+++ b/meta/conf/distro/debian-common.conf
@@ -5,6 +5,8 @@
BASE_DISTRO = "debian"
+HOST_BASE_DISTRO ?= "${BASE_DISTRO}"
+
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
DISTRO_APT_SOURCES ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}.list"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index f32d192e..14d876aa 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -29,6 +29,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = "${@https_support(d)}"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
+BOOTSTRAP_BASE_DISTRO = "${@d.getVar('HOST_BASE_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
FILESEXTRAPATHS_append = ":${BBPATH}"
inherit deb-dl-dir
@@ -111,7 +112,7 @@ def get_apt_source_mirror(d, aptsources_entry_list):
import re
if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- premirrors = "\S* file://${REPO_BASE_DIR}/${BASE_DISTRO}\n"
+ premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
else:
premirrors = d.getVar('DISTRO_APT_PREMIRRORS', True) or ""
mirror_list = [entry.split()
@@ -319,11 +320,15 @@ do_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
if [ -z "${BASE_REPO_KEY}" ]; then
line="[trusted=yes] ${line}"
fi
echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
mkdir -p ${ROOTFSDIR}/base-apt
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 506a28ff..05016d80 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -13,7 +13,10 @@ KEYFILES ?= ""
BASE_REPO_FEATURES ?= ""
populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ distro="${1}"
+ base_distro="${2}"
+
+ find "${DEBDIR}"/"${distro}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
# addition, md5sums are compared to ensure that the package is the
@@ -22,25 +25,25 @@ populate_base_apt() {
# Check if this package is already in base-apt
ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ repo_contains_package "${REPO_BASE_DIR}/${base_distro}" "${package}" ||
ret=$?
[ "${ret}" = "0" ] && continue
if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ repo_del_package "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
+ "${package}"
fi
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ repo_add_packages "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ find "${DEBSRCDIR}"/"${distro}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
@@ -55,20 +58,20 @@ repo() {
"${BASE_DISTRO_CODENAME}" \
"${WORKDIR}/distributions.in" \
"${KEYFILES}"
-
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
+ populate_base_apt "${DISTRO}" "${BASE_DISTRO}"
repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}"
+
+ if [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ repo_create "${REPO_BASE_DIR}"/"${HOST_BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${HOST_BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${WORKDIR}/distributions.in" \
+ "${KEYFILES}"
+ populate_base_apt "${HOST_DISTRO}" "${HOST_BASE_DISTRO}"
+ repo_sanity_test "${REPO_BASE_DIR}"/"${HOST_BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${HOST_BASE_DISTRO}"
+ fi
}
python do_cache() {
--
2.20.1
next prev parent reply other threads:[~2022-11-28 11:05 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-11-28 11:05 [PATCH v4 0/3] Fix cross-build from base-apt when custom DISTRO name used Uladzimir Bely
2022-11-28 11:05 ` Uladzimir Bely [this message]
2022-11-29 10:49 ` [PATCH v4 1/3] Populate base-apt from both DISTRO and HOST_DISTRO download dirs Jan Kiszka
2022-11-29 11:29 ` Uladzimir Bely
2022-11-29 12:07 ` Jan Kiszka
2022-11-28 11:05 ` [PATCH v4 2/3] testsuite: Run signed repro test in cross mode Uladzimir Bely
2022-11-28 11:05 ` [PATCH v4 3/3] testsuite: Replace bananapi with rpi-arm-v7 in signed repro tests Uladzimir Bely
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221128110534.31450-2-ubely@ilbers.de \
--to=ubely@ilbers.de \
--cc=isar-users@googlegroups.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox