Re: [PATCH v3 1/3] Populate base-apt from both DISTRO and HOST_DISTRO download dirs

[Jan Kiszka <jan.kiszka@siemens.com>]

On 25.11.22 07:42, Uladzimir Bely wrote:
> Downstreams may want to change DISTRO to some custom value.
> When cross-building, this leads to downloading packages for host
> and target distros to the different download subdirs.
> 
> While base-apt is populated only from DISTRO download subdir,
> second cached (ISAR_USE_CACHED_BASE_REPO = "1") build fails due to
> missing packages in base-apt.
> 
> The similar issue happens with cross-building raspberry targets
> since they use their own mirrors and DISTRO.
> 
> Fix the issue by splitting base-apt by introducing BASE_HOST_DISTRO
> variable that is used instead of BASE_DISTRO for caching builds
> for host-related components (e.g., isar-bootstrap, buildchroot-host,
> sbuild-chroot-host). Target components still use BASE_DISTRO.
> 
> Signed-off-by: Uladzimir Bely <ubely@ilbers.de>
> ---
>  meta-isar/conf/distro/raspbian-stretch.conf   |  2 +
>  meta-isar/conf/distro/raspios-bullseye.conf   |  2 +
>  meta-isar/conf/distro/ubuntu-focal.conf       |  2 +
>  meta/conf/distro/debian-common.conf           |  2 +
>  .../isar-bootstrap/isar-bootstrap.inc         |  9 +++-
>  meta/recipes-devtools/base-apt/base-apt.bb    | 47 ++++++++++---------
>  6 files changed, 40 insertions(+), 24 deletions(-)
> 
> diff --git a/meta-isar/conf/distro/raspbian-stretch.conf b/meta-isar/conf/distro/raspbian-stretch.conf
> index c8f523b3..66d965c9 100644
> --- a/meta-isar/conf/distro/raspbian-stretch.conf
> +++ b/meta-isar/conf/distro/raspbian-stretch.conf
> @@ -6,6 +6,8 @@
>  BASE_DISTRO = "raspbian"
>  BASE_DISTRO_CODENAME = "stretch"
>  
> +BASE_HOST_DISTRO = "debian"

Better prefix it with HOST, to be consistent:

HOST_BASE_DISTRO

> +
>  HOST_DISTRO ?= "debian-stretch"
>  HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
>  
> diff --git a/meta-isar/conf/distro/raspios-bullseye.conf b/meta-isar/conf/distro/raspios-bullseye.conf
> index 5e4a09c9..7bd81dd6 100644
> --- a/meta-isar/conf/distro/raspios-bullseye.conf
> +++ b/meta-isar/conf/distro/raspios-bullseye.conf
> @@ -6,6 +6,8 @@
>  BASE_DISTRO = "raspios"
>  BASE_DISTRO_arm64 = "debian"
>  
> +BASE_HOST_DISTRO ?= "debian"
> +
>  BASE_DISTRO_CODENAME = "bullseye"
>  HOST_DISTRO ?= "debian-bullseye"
>  HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
> diff --git a/meta-isar/conf/distro/ubuntu-focal.conf b/meta-isar/conf/distro/ubuntu-focal.conf
> index 379b5b30..75da79a1 100644
> --- a/meta-isar/conf/distro/ubuntu-focal.conf
> +++ b/meta-isar/conf/distro/ubuntu-focal.conf
> @@ -8,6 +8,8 @@ require conf/distro/debian-common.conf
>  BASE_DISTRO = "ubuntu"
>  BASE_DISTRO_CODENAME = "focal"
>  
> +BASE_HOST_DISTRO = "${BASE_DISTRO}"
> +
>  DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}-ports.list"
>  HOST_DISTRO_APT_SOURCES_arm64 ?= "conf/distro/${HOST_DISTRO}.list conf/distro/${HOST_DISTRO}-ports.list"
>  
> diff --git a/meta/conf/distro/debian-common.conf b/meta/conf/distro/debian-common.conf
> index c18eebc2..d7a13f4b 100644
> --- a/meta/conf/distro/debian-common.conf
> +++ b/meta/conf/distro/debian-common.conf
> @@ -5,6 +5,8 @@
>  
>  BASE_DISTRO = "debian"
>  
> +BASE_HOST_DISTRO ?= "${BASE_DISTRO}"
> +
>  HOST_DISTRO_APT_SOURCES ?= "conf/distro/${HOST_DISTRO}.list"
>  DISTRO_APT_SOURCES ?= "conf/distro/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}.list"
>  
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index f32d192e..2ee0d4cc 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -29,6 +29,7 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
>  DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = "${@https_support(d)}"
>  DISTRO_VARS_PREFIX ?= "${@'HOST_' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else ''}"
>  BOOTSTRAP_DISTRO = "${@d.getVar('HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'DISTRO')}"
> +BOOTSTRAP_BASE_DISTRO = "${@d.getVar('BASE_HOST_DISTRO' if d.getVar('BOOTSTRAP_FOR_HOST') == '1' else 'BASE_DISTRO')}"
>  FILESEXTRAPATHS_append = ":${BBPATH}"
>  
>  inherit deb-dl-dir
> @@ -111,7 +112,7 @@ def get_apt_source_mirror(d, aptsources_entry_list):
>      import re
>  
>      if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
> -        premirrors = "\S* file://${REPO_BASE_DIR}/${BASE_DISTRO}\n"
> +        premirrors = "\S* file://${REPO_BASE_DIR}/${BOOTSTRAP_BASE_DISTRO}\n"
>      else:
>          premirrors = d.getVar('DISTRO_APT_PREMIRRORS', True) or ""
>      mirror_list = [entry.split()
> @@ -319,11 +320,15 @@ do_bootstrap() {
>                           "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
>          mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
>          if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
> -            line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
> +            line="file:///base-apt/${BOOTSTRAP_BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
>              if [ -z "${BASE_REPO_KEY}" ]; then
>                  line="[trusted=yes] ${line}"
>              fi
>              echo "deb ${line}" >  "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
> +            line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"

Won't this cause duplicate lines (with complaints of apt) in case of
BOOTSTRAP_BASE_DISTRO == BASE_DISTRO?

> +            if [ -z "${BASE_REPO_KEY}" ]; then
> +                line="[trusted=yes] ${line}"
> +            fi
>              echo "deb-src ${line}" >>  "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
>  
>              mkdir -p ${ROOTFSDIR}/base-apt
> diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
> index 506a28ff..e928cd52 100644
> --- a/meta/recipes-devtools/base-apt/base-apt.bb
> +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> @@ -13,7 +13,10 @@ KEYFILES ?= ""
>  BASE_REPO_FEATURES ?= ""
>  
>  populate_base_apt() {
> -    find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
> +    distro="${1}"
> +    base_distro="${2}"
> +
> +    find "${DEBDIR}"/"${distro}" -name '*\.deb' | while read package; do
>          # NOTE: due to packages stored by reprepro are not modified, we can
>          # use search by filename to check if package is already in repo. In
>          # addition, md5sums are compared to ensure that the package is the
> @@ -22,25 +25,25 @@ populate_base_apt() {
>  
>          # Check if this package is already in base-apt
>          ret=0
> -        repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
> +        repo_contains_package "${REPO_BASE_DIR}/${base_distro}" "${package}" ||
>              ret=$?
>          [ "${ret}" = "0" ] && continue
>          if [ "${ret}" = "1" ]; then
> -            repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> -                "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> +            repo_del_package "${REPO_BASE_DIR}"/"${base_distro}" \
> +                "${REPO_BASE_DB_DIR}"/"${base_distro}" \
>                  "${BASE_DISTRO_CODENAME}" \
> -                "${base_apt_p}"
> +                "${package}"
>          fi
>  
> -        repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> -            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> +        repo_add_packages "${REPO_BASE_DIR}"/"${base_distro}" \
> +            "${REPO_BASE_DB_DIR}"/"${base_distro}" \
>              "${BASE_DISTRO_CODENAME}" \
>              "${package}"
>      done
>  
> -    find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
> -        repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> -            "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> +    find "${DEBSRCDIR}"/"${distro}" -name '*\.dsc' | while read package; do
> +        repo_add_srcpackage "${REPO_BASE_DIR}"/"${base_distro}" \
> +            "${REPO_BASE_DB_DIR}"/"${base_distro}" \
>              "${BASE_DISTRO_CODENAME}" \
>              "${package}"
>      done
> @@ -55,20 +58,20 @@ repo() {
>          "${BASE_DISTRO_CODENAME}" \
>          "${WORKDIR}/distributions.in" \
>          "${KEYFILES}"
> -
> -    if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> -        [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> -        # We would need two separate repository paths for that.
> -        # Otherwise packages (especially the 'all' arch ones) from one
> -        # distribution can influence the package versions of the other
> -        # distribution.
> -        bbfatal "Different host and target distributions are currently not supported." \
> -                "Try it without cross-build."
> -    fi
> -
> -    populate_base_apt
> +    populate_base_apt "${DISTRO}" "${BASE_DISTRO}"
>      repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
>          "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}"
> +
> +    if [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> +        repo_create "${REPO_BASE_DIR}"/"${BASE_HOST_DISTRO}" \
> +            "${REPO_BASE_DB_DIR}"/"${BASE_HOST_DISTRO}" \
> +            "${BASE_DISTRO_CODENAME}" \
> +            "${WORKDIR}/distributions.in" \
> +            "${KEYFILES}"
> +        populate_base_apt "${HOST_DISTRO}" "${BASE_HOST_DISTRO}"
> +        repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_HOST_DISTRO}" \
> +            "${REPO_BASE_DB_DIR}"/"${BASE_HOST_DISTRO}"
> +    fi
>  }
>  
>  python do_cache() {

Looks promising!

Jan

-- 
Siemens AG, Technology
Competence Center Embedded Linux