[PATCH v2 1/7] Implement support for setting up the local apt mirror and isar repository with aptly.

[Benedikt Niedermayr <benbrenson89@googlemail.com>]

This patch introduces support for basically setting up two things:
- A local debian mirror for mirroring upstream repositories.
- A local isar repo, which holds packages created by Isar.

The mirror is getting initialized before buildchroot and image rootfs where created.

All required packages where extracted from recipes with the get_filter_list() function.
This function will run an own parser in order to take all possible append files into account.

Runtime dependencies:
- All packages defined with BUILDCHROOT_PREINSTALL, IMAGE_PREINSTALL and DEBIAN_DEPENDS within Isar
  where collected and stored in a format aptly understands.

Buildtime dependencies:
- The get_filter_list() searches for debian control files at ${WORKDIR}/${SRC_DIR}/debian/control,
  and parses out the Build-Depends.
- The control file parsing is implemented in "deb822.py" module which is part of the "python3-debian" package.

Todos:
- Creating mirrors for different sections is hardcoded for now. This should be fixed in future since not
  all distibutions refer to the same remote sections(e.g. raspbian).
- Find a clean naming convention for mirrors, repos and snapshots, in order to get multiconfig compatible.
- Extracting dependencies when inheriting from dpkg-raw, is not working. Since dpkg-raw currently doesn't support any Build-Depends.

Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
 meta/recipes-devtools/isar-apt/files/aptly.conf.in |  17 ++
 meta/recipes-devtools/isar-apt/isar-apt.bb         | 174 ++++++++++++++++++---
 2 files changed, 170 insertions(+), 21 deletions(-)
 create mode 100644 meta/recipes-devtools/isar-apt/files/aptly.conf.in

diff --git a/meta/recipes-devtools/isar-apt/files/aptly.conf.in b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
new file mode 100644
index 0000000..73e78f1
--- /dev/null
+++ b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
@@ -0,0 +1,17 @@
+{
+  "rootDir": "##CACHE_DIR##",
+  "downloadConcurrency": 6,
+  "downloadSpeedLimit": 0,
+  "architectures": [],
+  "dependencyFollowSuggests": false,
+  "dependencyFollowRecommends": true,
+  "dependencyFollowAllVariants": false,
+  "dependencyFollowSource": false,
+  "gpgDisableSign": true,
+  "gpgDisableVerify": true,
+  "downloadSourcePackages": false,
+  "ppaDistributorID": "ubuntu",
+  "ppaCodename": "",
+  "S3PublishEndpoints": {},
+  "SwiftPublishEndpoints": {}
+}
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index 9c31d12..0342227 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,29 +1,161 @@
-# This software is a part of ISAR.
-# Copyright (C) 2015-2017 ilbers GmbH
+inherit apt-cache
 
-SRC_URI = "file://distributions.in"
+SRC_URI = "file://aptly.conf.in \
+          "
 
-CACHE_CONF_DIR = "${DEPLOY_DIR_APT}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${DEPLOY_DIR_APT}/isar.lock"
-
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
+# Setup the apt cache, load snapshot of last build.
 do_cache_config() {
-    if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
-        sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
-            ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
-    fi
 
-    path_cache="${DEPLOY_DIR_APT}/${DISTRO}"
-    path_databases="${DEPLOY_DIR_DB}/${DISTRO}"
+    # Very first build of isar. Setting up the cache.
+    if [ ! -e "${CACHE_CONF_DIR}/aptly.conf" ]; then
+        bbwarn "Setting up local apt mirror"
+
+        # Check if aptly is installed
+        aptly version || bbfatal "Aptly is not installed. Please install aptly (>= 1.2)"
+
+        sed -e 's|##CACHE_DIR##|${TOPDIR}/apt-cache/${DISTRO}|' \
+            ${WORKDIR}/aptly.conf.in > ${CACHE_CONF_DIR}/aptly.conf || \
+            rm ${CACHE_CONF_DIR}/aptly.conf
+
+        # Setup Isar repo
+        repo_db_create ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+        repo_db_publish ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+
+        # Setup mirrors
+        pre_fetch="Essential (yes)|${@get_filter_list(d)}"
+        bbnote "Prefetch filter: $pre_fetch"
+
+        mirror_create "$pre_fetch" ${MIRROR} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}
+        mirror_create "$pre_fetch" ${MIRROR_SECURITY} ${DISTRO_APT_SOURCE_SEC} ${DISTRO_SUITE}/updates
+        mirror_create "$pre_fetch" ${MIRROR_UPDATES} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}-updates
+
+        mirror_update ${MIRROR}
+        mirror_update ${MIRROR_SECURITY}
+        mirror_update ${MIRROR_UPDATES}
+
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME} ${MIRROR}
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-updates ${MIRROR_UPDATES}
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-security ${MIRROR_SECURITY}
 
-    if [ ! -d "${path_databases}" ]; then
-        reprepro -b ${path_cache} \
-                 --dbdir ${path_databases} \
-                 export ${DEBDISTRONAME}
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME} ${ISAR_MIRROR_PREFIX} ${DISTRO_SUITE}
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-updates ${ISAR_MIRROR_PREFIX}-updates ${DISTRO_SUITE}
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-security ${ISAR_MIRROR_PREFIX}-security ${DISTRO_SUITE}
     fi
-}
 
+    if [ -e "${ISAR_FIRST_BUILD_DONE}" ] && [ "${REPRODUCIBLE_BUILD_ENABLED}" == "1" ]; then
+        cache_load_snapshot ${SNAPSHOT_BASENAME}_${MACHINE}
+    fi
+}
 addtask cache_config after do_unpack before do_build
+do_cache_config[dirs] += "${CACHE_CONF_DIR}"
+do_cache_config[stamp-extra-info] = "${DISTRO}"
+do_cache_config[lockfiles] = "${TOPDIR}/apt-cache/isar.lock"
+
+
+# Return a string containing all packages required for setting up the mirror.
+# Implementing and running an own parser will also ensure considering bbappend files.
+def get_filter_list(d):
+    import sys
+    import bb.cooker, bb.cookerdata
+
+    #
+    # Todo: Move class into lib/oe
+    #
+    class DummyConfigParameters(bb.cookerdata.ConfigParameters):
+        """ Class for generating Dummy config parameters. Required for creating valid
+            cookerdata.ConfigParameters and setting these within cookerdata.CookerConfiguration
+            with cookerdata.CookerConfiguration.setConfigParameters().
+        """
+        def __init__(self, **options):
+            self.initial_options = options
+            super(DummyConfigParameters, self).__init__()
+
+        def parseCommandLine(self, argv=sys.argv):
+            class DummyOptions:
+                def __init__(self, initial_options):
+                    for key, val in initial_options.items():
+                        setattr(self, key, val)
+
+            return DummyOptions(self.initial_options), None
+
+
+    # Get collections
+    # A collection object stores information and methods about bbfiles as well as a list
+    # of bbfiles.
+    bbfile_config_priorities = []
+    collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
+    bbfiles, masked, searchdirs = collection.collect_bbfiles(d, d)
+
+    configparams = DummyConfigParameters()
+
+    # Parse of configuration data for a recipes' environment
+    configuration = bb.cookerdata.CookerConfiguration()
+    configuration.setConfigParameters(configparams)
+
+    # Get the databuilder, which is responsible for holding
+    # config related information.
+    databuilder = bb.cookerdata.CookerDataBuilder(configuration, False)
+    databuilder.parseBaseConfiguration()
+    parser = bb.cache.NoCache(databuilder)
+
+    prefetch_packages = ''
+
+    for bbfile in bbfiles:
+        appendfiles = collection.get_file_appends(bbfile)
+        bb.note('Loading data from {}'.format(bbfile))
+        data = parser.loadDataFull(bbfile, appendfiles)
+
+        buildchroot_preinstall = data.getVar('BUILDCHROOT_PREINSTALL', True) or ''
+        image_preinstall = data.getVar('IMAGE_PREINSTALL', True) or ''
+        debian_depends = data.getVar('DEBIAN_DEPENDS', True) or ''
+
+        # Runtime dependencies
+        buildchroot_preinstall = '|'.join(buildchroot_preinstall.split())
+        image_preinstall = '|'.join(image_preinstall.split())
+        debian_depends = debian_depends.replace(',', '|')
+
+        if buildchroot_preinstall:
+            prefetch_packages += buildchroot_preinstall + '|'
+        if image_preinstall:
+            prefetch_packages += image_preinstall + '|'
+        if debian_depends:
+            prefetch_packages += debian_depends + '|'
+
+        # Buildtime dependencies
+        workdir = data.getVar('WORKDIR', True) or ''
+        src_dir = data.getVar('SRC_DIR', True) or ''
+        control = os.path.join(workdir, src_dir, 'debian', 'control')
+
+        # Build-Depends
+        for dep in iter_deps(control, 'Build-Depends'):
+            bb.note('Found Build-Depends:%s in %s' % (dep, data.getVar('P')))
+            prefetch_packages += dep + '|'
+
+        # Build-Depends-Indep
+        for dep in iter_deps(control, 'Build-Depends-Indep'):
+            bb.note('Found Build-Depends-Indep:%s in %s' % (dep, data.getVar('P')))
+            prefetch_packages += dep + '|'
+
+    return prefetch_packages.strip('|')
+
+
+
+# Generator for iterating over package dependencies
+# Strings of type "pkg-name ([<>=] ${*:*})" are not supported,
+# they are returned as raw strings.
+def iter_deps(control, field):
+    import deb822
+
+    try:
+        f = open(control)
+    except:
+        return
+
+    for paragraph in deb822.Deb822.iter_paragraphs(f):
+        for item in paragraph.items():
+            if item[0] != field:
+                continue
+            archs = deb822.PkgRelation.parse_relations(item[1])
+            for name in archs:
+                for i in name:
+                    yield(i['name'])
-- 
2.7.4