Re: [PATCH 1/1] deduplicate apt sources.list based on filenames

[Gylstorff Quirin <quirin.gylstorff@siemens.com>]

On 7/26/23 08:14, 'Felix Moessbauer' via isar-users wrote:
> When combining layers, it can happen that the same DISTRO_APT_SOURCES
> entry is added multiple times. This creates duplicate entries in the
> bootstrap.list, which is considered invalid by apt. To solve this, we
> deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable.
> In addition, this also avoids adding the same SRC_URI entry twice.
> 
> Note, that this only solves the trivial case that the duplicate entries
> come from the same file. However in case duplicate entries are added
> from multiple files with different names, the :remove syntax can be
> used to remove these for the affected targets.
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>   meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++-------
>   1 file changed, 6 insertions(+), 7 deletions(-)
> 
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index b94ae0bd..8af73a9b 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -59,8 +59,8 @@ python () {
>           filename = os.path.relpath(fetcher.localpath(key), topdir)
>           d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
>   
> -    distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or ""
> -    for file in distro_apt_sources.split():
> +    distro_apt_sources = get_aptsources_list(d)
> +    for file in distro_apt_sources:
>           d.appendVar("SRC_URI", " file://%s" % file)
>   
>       distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
> @@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, file_out):
>   
>   def get_aptsources_list(d):
>       import errno
> +    from collections import OrderedDict
>       apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
> -    list = (d.getVar(apt_sources_var) or "").split()
> +    apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
Why not use `set()` if the goal is to remove all duplicates?

Quirin
>       ret = []
> -    for p in list:
> +    for p in apt_sources_list:
>           try:
>               f = bb.parse.resolve_file(p, d)
>               ret.append(f)
> @@ -251,9 +252,7 @@ python do_apt_config_prepare() {
>   
>       apt_sources_out = d.getVar("APTSRCS")
>       apt_sources_init_out = d.getVar("APTSRCS_INIT")
> -    apt_sources_list = (
> -        d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or ""
> -    ).split()
> +    apt_sources_list = get_aptsources_list(d)
>   
>       aggregate_files(d, apt_sources_list, apt_sources_init_out)
>       aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)