[PATCH 1/1] deduplicate apt sources.list based on filenames

[Felix Moessbauer <felix.moessbauer@siemens.com>]

When combining layers, it can happen that the same DISTRO_APT_SOURCES
entry is added multiple times. This creates duplicate entries in the
bootstrap.list, which is considered invalid by apt. To solve this, we
deduplicate the values in the [HOST_]DISTRO_APT_SOURCES variable.
In addition, this also avoids adding the same SRC_URI entry twice.

Note, that this only solves the trivial case that the duplicate entries
come from the same file. However in case duplicate entries are added
from multiple files with different names, the :remove syntax can be
used to remove these for the affected targets.

Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
---
 meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index b94ae0bd..8af73a9b 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -59,8 +59,8 @@ python () {
         filename = os.path.relpath(fetcher.localpath(key), topdir)
         d.appendVar("THIRD_PARTY_APT_KEYFILES", " ${TOPDIR}/%s" % filename)
 
-    distro_apt_sources = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or ""
-    for file in distro_apt_sources.split():
+    distro_apt_sources = get_aptsources_list(d)
+    for file in distro_apt_sources:
         d.appendVar("SRC_URI", " file://%s" % file)
 
     distro_apt_preferences = d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_PREFERENCES") or ""
@@ -149,10 +149,11 @@ def aggregate_aptsources_list(d, file_list, file_out):
 
 def get_aptsources_list(d):
     import errno
+    from collections import OrderedDict
     apt_sources_var = d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES"
-    list = (d.getVar(apt_sources_var) or "").split()
+    apt_sources_list = list(OrderedDict.fromkeys((d.getVar(apt_sources_var) or "").split()))
     ret = []
-    for p in list:
+    for p in apt_sources_list:
         try:
             f = bb.parse.resolve_file(p, d)
             ret.append(f)
@@ -251,9 +252,7 @@ python do_apt_config_prepare() {
 
     apt_sources_out = d.getVar("APTSRCS")
     apt_sources_init_out = d.getVar("APTSRCS_INIT")
-    apt_sources_list = (
-        d.getVar(d.getVar("DISTRO_VARS_PREFIX") + "DISTRO_APT_SOURCES") or ""
-    ).split()
+    apt_sources_list = get_aptsources_list(d)
 
     aggregate_files(d, apt_sources_list, apt_sources_init_out)
     aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
-- 
2.34.1