* [PATCH v2 1/3] Populate base-apt from both DISTRO and HOST_DISTRO download dirs
2022-11-25 6:36 [PATCH v2 0/3] Fix cross-build from base-apt when custom DISTRO name used Uladzimir Bely
@ 2022-11-25 6:36 ` Uladzimir Bely
2022-11-25 6:36 ` [PATCH v2 2/3] testsuite: Run signed repro test in cross mode Uladzimir Bely
2022-11-25 6:36 ` [PATCH v2 3/3] testsuite: Replace bananapi with rpi-arm-v7 in signed repro tests Uladzimir Bely
2 siblings, 0 replies; 4+ messages in thread
From: Uladzimir Bely @ 2022-11-25 6:36 UTC (permalink / raw)
To: isar-users
Downstreams may want to change DISTRO to some custom value.
When cross-building, this leads to downloading packages for host
and target distros to the different download subdirs.
While base-apt is populated only from DISTRO download subdir,
second cached (ISAR_USE_CACHED_BASE_REPO = "1") build fails due to
missing packages in base-apt.
The similar issue happens with cross-building raspberry targets
since they use their own mirrors and DISTRO.
Fix the issue by splitting base-apt by introducing BASE_HOST_DISTRO
variable that is used instead of BASE_DISTRO for caching builds
for host-related components (e.g., isar-bootstrap, buildchroot-host,
sbuild-chroot-host). Target components still use BASE_DISTRO.
Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
---
meta-isar/conf/distro/raspbian-stretch.conf | 2 +
meta-isar/conf/distro/raspios-bullseye.conf | 2 +
meta-isar/conf/distro/ubuntu-focal.conf | 2 +
meta/conf/distro/debian-common.conf | 2 +
.../isar-bootstrap/isar-bootstrap.inc | 9 +++-
meta/recipes-devtools/base-apt/base-apt.bb | 47 ++++++++++---------
6 files changed, 40 insertions(+), 24 deletions(-)
diff --git a/meta-isar/conf/distro/raspbian-stretch.conf b/meta-isar/conf/distro/raspbian-stretch.conf
index c8f523b3..66d965c9 100644
--- a/meta-isar/conf/distro/raspbian-stretch.conf
+++ b/meta-isar/conf/distro/raspbian-stretch.conf
@@ -6,6 +6,8 @@
BASE_DISTRO = "raspbian"
BASE_DISTRO_CODENAME = "stretch"
+BASE_HOST_DISTRO = "debian"
+
HOST_DISTRO ?= "debian-stretch"
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
diff --git a/meta-isar/conf/distro/raspios-bullseye.conf b/meta-isar/conf/distro/raspios-bullseye.conf
index 5e4a09c9..7bd81dd6 100644
--- a/meta-isar/conf/distro/raspios-bullseye.conf
+++ b/meta-isar/conf/distro/raspios-bullseye.conf
@@ -6,6 +6,8 @@
BASE_DISTRO = "raspios"
BASE_DISTRO_arm64 = "debian"
+BASE_HOST_DISTRO ?= "debian"
+
BASE_DISTRO_CODENAME = "bullseye"
HOST_DISTRO ?= "debian-bullseye"
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
index 379b5b30..75da79a1 100644
--- a/meta-isar/conf/distro/ubuntu-focal.conf
+++ b/meta-isar/conf/distro/ubuntu-focal.conf
@@ -8,6 +8,8 @@ require conf/distro/debian-common.conf
BASE_DISTRO = "ubuntu"
BASE_DISTRO_CODENAME = "focal"
+BASE_HOST_DISTRO = "${BASE_DISTRO}"
+
DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}-ports.list"
HOST_DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${HOST_DISTRO}.list conf/distro/${HOST_DISTRO}-ports.list"
diff --git a/meta/conf/distro/debian-common.conf b/meta/conf/distro/debian-common.conf
index c18eebc2..d7a13f4b 100644
--- a/meta/conf/distro/debian-common.conf
+++ b/meta/conf/distro/debian-common.conf
@@ -5,6 +5,8 @@
BASE_DISTRO = "debian"
+BASE_HOST_DISTRO ?= "${BASE_DISTRO}"
+
HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
DISTRO_APT_SOURCES ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}.list"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index f32d192e..2ee0d4cc 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -29,6 +29,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = "${@https_support(d)}"
DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
+BOOTSTRAP_BASE_DISTRO = "${@d.getVar('BASE_HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
FILESEXTRAPATHS_append = ":${BBPATH}"
inherit deb-dl-dir
@@ -111,7 +112,7 @@ def get_apt_source_mirror(d, aptsources_entry_list):
import re
if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
- premirrors = "\S* file://${REPO_BASE_DIR}/${BASE_DISTRO}\n"
+ premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
else:
premirrors = d.getVar('DISTRO_APT_PREMIRRORS', True) or ""
mirror_list = [entry.split()
@@ -319,11 +320,15 @@ do_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
if [ -z "${BASE_REPO_KEY}" ]; then
line="[trusted=yes] ${line}"
fi
echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
mkdir -p ${ROOTFSDIR}/base-apt
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 506a28ff..e928cd52 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -13,7 +13,10 @@ KEYFILES ?= ""
BASE_REPO_FEATURES ?= ""
populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ distro="${1}"
+ base_distro="${2}"
+
+ find "${DEBDIR}"/"${distro}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
# addition, md5sums are compared to ensure that the package is the
@@ -22,25 +25,25 @@ populate_base_apt() {
# Check if this package is already in base-apt
ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ repo_contains_package "${REPO_BASE_DIR}/${base_distro}" "${package}" ||
ret=$?
[ "${ret}" = "0" ] && continue
if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ repo_del_package "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
+ "${package}"
fi
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ repo_add_packages "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ find "${DEBSRCDIR}"/"${distro}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${base_distro}" \
+ "${REPO_BASE_DB_DIR}"/"${base_distro}" \
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
@@ -55,20 +58,20 @@ repo() {
"${BASE_DISTRO_CODENAME}" \
"${WORKDIR}/distributions.in" \
"${KEYFILES}"
-
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
+ populate_base_apt "${DISTRO}" "${BASE_DISTRO}"
repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}"
+
+ if [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ repo_create "${REPO_BASE_DIR}"/"${BASE_HOST_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_HOST_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${WORKDIR}/distributions.in" \
+ "${KEYFILES}"
+ populate_base_apt "${HOST_DISTRO}" "${BASE_HOST_DISTRO}"
+ repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_HOST_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_HOST_DISTRO}"
+ fi
}
python do_cache() {
--
2.20.1
^ permalink raw reply [flat|nested] 4+ messages in thread