[PATCH 1/6] Implement support for setting up the local apt mirror and isar repository with aptly.

[Benedikt Niedermayr <benbrenson89@googlemail.com>]

This patch introduces support for basically setting up two things:
- A local debian mirror for mirroring upstream repositories.
- A local isar repo, which holds packages created by Isar.

The mirror is getting initialized before buildchroot and image rootfs where created.

All required packages where extracted from recipes with the get_filter_list() function.
This function will run an own parser in order to take all possible append files into account.
All packages defined with BUILDCHROOT_PREINSTALL, IMAGE_PREINSTALL and DEBIAN_DEPENDS within Isar
where collected and stored in a format aptly understands.

Todos:
- Creating mirrors for different sections is hardcoded for now. This should be fixed in future since not
  all distibutions refer to the same remote sections(e.g. raspbian).
- Find a clean naming convention for mirrors, repos and snapshots, in order to get multiconfig compatible.

Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
 meta/recipes-devtools/isar-apt/files/aptly.conf.in |  17 +++
 meta/recipes-devtools/isar-apt/isar-apt.bb         | 131 +++++++++++++++++----
 2 files changed, 127 insertions(+), 21 deletions(-)
 create mode 100644 meta/recipes-devtools/isar-apt/files/aptly.conf.in

diff --git a/meta/recipes-devtools/isar-apt/files/aptly.conf.in b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
new file mode 100644
index 0000000..73e78f1
--- /dev/null
+++ b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
@@ -0,0 +1,17 @@
+{
+  "rootDir": "##CACHE_DIR##",
+  "downloadConcurrency": 6,
+  "downloadSpeedLimit": 0,
+  "architectures": [],
+  "dependencyFollowSuggests": false,
+  "dependencyFollowRecommends": true,
+  "dependencyFollowAllVariants": false,
+  "dependencyFollowSource": false,
+  "gpgDisableSign": true,
+  "gpgDisableVerify": true,
+  "downloadSourcePackages": false,
+  "ppaDistributorID": "ubuntu",
+  "ppaCodename": "",
+  "S3PublishEndpoints": {},
+  "SwiftPublishEndpoints": {}
+}
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index 9c31d12..f1d38bd 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,29 +1,118 @@
-# This software is a part of ISAR.
-# Copyright (C) 2015-2017 ilbers GmbH
+inherit apt-cache
 
-SRC_URI = "file://distributions.in"
+# Setup the apt cache, load snapshot of last build.
+do_cache_config() {
 
-CACHE_CONF_DIR = "${DEPLOY_DIR_APT}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${DEPLOY_DIR_APT}/isar.lock"
+    # Very first build of isar. Setting up the cache.
+    if [ ! -e "${CACHE_CONF_DIR}/aptly.conf" ]; then
+        bbwarn "Setting up local apt mirror"
 
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-do_cache_config() {
-    if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
-        sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
-            ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
-    fi
+        # Check if aptly is installed
+        aptly version || bbfatal "Aptly is not installed. Please install aptly (>= 1.2)"
+
+        sed -e 's|##CACHE_DIR##|${TOPDIR}/apt-cache/${DISTRO}|' \
+            ${FILESDIR}/aptly.conf.in > ${CACHE_CONF_DIR}/aptly.conf || \
+            rm ${CACHE_CONF_DIR}/aptly.conf
+
+        # Setup Isar repo
+        repo_db_create ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+        repo_db_publish ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+
+        # Setup mirrors
+        pre_fetch="${@get_filter_list(d)}"
 
-    path_cache="${DEPLOY_DIR_APT}/${DISTRO}"
-    path_databases="${DEPLOY_DIR_DB}/${DISTRO}"
+        mirror_create "$pre_fetch" ${MIRROR} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}
+        mirror_create "$pre_fetch" ${MIRROR_SECURITY} ${DISTRO_APT_SOURCE_SEC} ${DISTRO_SUITE}/updates
+        mirror_create "$pre_fetch" ${MIRROR_UPDATES} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}-updates
 
-    if [ ! -d "${path_databases}" ]; then
-        reprepro -b ${path_cache} \
-                 --dbdir ${path_databases} \
-                 export ${DEBDISTRONAME}
+        mirror_update ${MIRROR}
+        mirror_update ${MIRROR_SECURITY}
+        mirror_update ${MIRROR_UPDATES}
+
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME} ${MIRROR}
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-updates ${MIRROR_UPDATES}
+        snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-security ${MIRROR_SECURITY}
+
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME} ${ISAR_MIRROR_PREFIX} ${DISTRO_SUITE}
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-updates ${ISAR_MIRROR_PREFIX}-updates ${DISTRO_SUITE}
+        snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-security ${ISAR_MIRROR_PREFIX}-security ${DISTRO_SUITE}
     fi
-}
 
+    if [ -e "${ISAR_FIRST_BUILD_DONE}" ] && [ "${REPRODUCIBLE_BUILD_ENABLED}" == "1" ]; then
+        cache_load_snapshot ${SNAPSHOT_BASENAME}_${MACHINE}
+    fi
+}
 addtask cache_config after do_unpack before do_build
+do_cache_config[dirs] += "${CACHE_CONF_DIR}"
+do_cache_config[stamp-extra-info] = "${DISTRO}"
+do_cache_config[lockfiles] = "${TOPDIR}/apt-cache/isar.lock"
+
+
+# Return a string containing all packages required for setting up the mirror.
+# Implementing and running an own parser will also ensure considering bbappend files.
+def get_filter_list(d):
+    import sys
+    import bb.cooker, bb.cookerdata
+
+    #
+    # Todo: Move class into lib/oe
+    #
+    class DummyConfigParameters(bb.cookerdata.ConfigParameters):
+        """ Class for generating Dummy config parameters. Required for creating valid
+            cookerdata.ConfigParameters and setting these within cookerdata.CookerConfiguration
+            with cookerdata.CookerConfiguration.setConfigParameters().
+        """
+        def __init__(self, **options):
+            self.initial_options = options
+            super(DummyConfigParameters, self).__init__()
+
+        def parseCommandLine(self, argv=sys.argv):
+            class DummyOptions:
+                def __init__(self, initial_options):
+                    for key, val in initial_options.items():
+                        setattr(self, key, val)
+
+            return DummyOptions(self.initial_options), None
+
+
+    # Get collections
+    # A collection object stores information and methods about bbfiles as well as a list
+    # of bbfiles.
+    bbfile_config_priorities = []
+    collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
+    bbfiles, masked, searchdirs = collection.collect_bbfiles(d, d)
+
+    configparams = DummyConfigParameters()
+
+    # Parse of configuration data for a recipes' environment
+    configuration = bb.cookerdata.CookerConfiguration()
+    configuration.setConfigParameters(configparams)
+
+    # Get the databuilder, which is responsible for holding
+    # config related information.
+    databuilder = bb.cookerdata.CookerDataBuilder(configuration, False)
+    databuilder.parseBaseConfiguration()
+    parser = bb.cache.NoCache(databuilder)
+
+    prefetch_packages = ''
+
+    for bbfile in bbfiles:
+        appendfiles = collection.get_file_appends(bbfile)
+        bb.note('Loading data from {}'.format(bbfile))
+        data = parser.loadDataFull(bbfile, appendfiles)
+
+        buildchroot_preinstall = data.getVar('BUILDCHROOT_PREINSTALL', True) or ''
+        image_preinstall = data.getVar('IMAGE_PREINSTALL', True) or ''
+        debian_depends = data.getVar('DEBIAN_DEPENDS', True) or ''
+
+        buildchroot_preinstall = '|'.join(buildchroot_preinstall.split())
+        image_preinstall = '|'.join(image_preinstall.split())
+        debian_depends = debian_depends.replace(',', '|')
+
+        if buildchroot_preinstall:
+            prefetch_packages += buildchroot_preinstall + '|'
+        if image_preinstall:
+            prefetch_packages += image_preinstall + '|'
+        if debian_depends:
+            prefetch_packages += debian_depends + '|'
+    return prefetch_packages.strip('|')
-- 
2.7.4