* [PATCH v2 1/7] Implement support for setting up the local apt mirror and isar repository with aptly.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 2/7] Added API class for apt cache Benedikt Niedermayr
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
This patch introduces support for basically setting up two things:
- A local debian mirror for mirroring upstream repositories.
- A local isar repo, which holds packages created by Isar.
The mirror is getting initialized before buildchroot and image rootfs where created.
All required packages where extracted from recipes with the get_filter_list() function.
This function will run an own parser in order to take all possible append files into account.
Runtime dependencies:
- All packages defined with BUILDCHROOT_PREINSTALL, IMAGE_PREINSTALL and DEBIAN_DEPENDS within Isar
where collected and stored in a format aptly understands.
Buildtime dependencies:
- The get_filter_list() searches for debian control files at ${WORKDIR}/${SRC_DIR}/debian/control,
and parses out the Build-Depends.
- The control file parsing is implemented in "deb822.py" module which is part of the "python3-debian" package.
Todos:
- Creating mirrors for different sections is hardcoded for now. This should be fixed in future since not
all distibutions refer to the same remote sections(e.g. raspbian).
- Find a clean naming convention for mirrors, repos and snapshots, in order to get multiconfig compatible.
- Extracting dependencies when inheriting from dpkg-raw, is not working. Since dpkg-raw currently doesn't support any Build-Depends.
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/recipes-devtools/isar-apt/files/aptly.conf.in | 17 ++
meta/recipes-devtools/isar-apt/isar-apt.bb | 174 ++++++++++++++++++---
2 files changed, 170 insertions(+), 21 deletions(-)
create mode 100644 meta/recipes-devtools/isar-apt/files/aptly.conf.in
diff --git a/meta/recipes-devtools/isar-apt/files/aptly.conf.in b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
new file mode 100644
index 0000000..73e78f1
--- /dev/null
+++ b/meta/recipes-devtools/isar-apt/files/aptly.conf.in
@@ -0,0 +1,17 @@
+{
+ "rootDir": "##CACHE_DIR##",
+ "downloadConcurrency": 6,
+ "downloadSpeedLimit": 0,
+ "architectures": [],
+ "dependencyFollowSuggests": false,
+ "dependencyFollowRecommends": true,
+ "dependencyFollowAllVariants": false,
+ "dependencyFollowSource": false,
+ "gpgDisableSign": true,
+ "gpgDisableVerify": true,
+ "downloadSourcePackages": false,
+ "ppaDistributorID": "ubuntu",
+ "ppaCodename": "",
+ "S3PublishEndpoints": {},
+ "SwiftPublishEndpoints": {}
+}
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index 9c31d12..0342227 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,29 +1,161 @@
-# This software is a part of ISAR.
-# Copyright (C) 2015-2017 ilbers GmbH
+inherit apt-cache
-SRC_URI = "file://distributions.in"
+SRC_URI = "file://aptly.conf.in \
+ "
-CACHE_CONF_DIR = "${DEPLOY_DIR_APT}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${DEPLOY_DIR_APT}/isar.lock"
-
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
+# Setup the apt cache, load snapshot of last build.
do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
- path_cache="${DEPLOY_DIR_APT}/${DISTRO}"
- path_databases="${DEPLOY_DIR_DB}/${DISTRO}"
+ # Very first build of isar. Setting up the cache.
+ if [ ! -e "${CACHE_CONF_DIR}/aptly.conf" ]; then
+ bbwarn "Setting up local apt mirror"
+
+ # Check if aptly is installed
+ aptly version || bbfatal "Aptly is not installed. Please install aptly (>= 1.2)"
+
+ sed -e 's|##CACHE_DIR##|${TOPDIR}/apt-cache/${DISTRO}|' \
+ ${WORKDIR}/aptly.conf.in > ${CACHE_CONF_DIR}/aptly.conf || \
+ rm ${CACHE_CONF_DIR}/aptly.conf
+
+ # Setup Isar repo
+ repo_db_create ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+ repo_db_publish ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+
+ # Setup mirrors
+ pre_fetch="Essential (yes)|${@get_filter_list(d)}"
+ bbnote "Prefetch filter: $pre_fetch"
+
+ mirror_create "$pre_fetch" ${MIRROR} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}
+ mirror_create "$pre_fetch" ${MIRROR_SECURITY} ${DISTRO_APT_SOURCE_SEC} ${DISTRO_SUITE}/updates
+ mirror_create "$pre_fetch" ${MIRROR_UPDATES} ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}-updates
+
+ mirror_update ${MIRROR}
+ mirror_update ${MIRROR_SECURITY}
+ mirror_update ${MIRROR_UPDATES}
+
+ snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME} ${MIRROR}
+ snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-updates ${MIRROR_UPDATES}
+ snapshot_create_from_mirror ${MIRROR_SNAPSHOT_BASENAME}-security ${MIRROR_SECURITY}
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
+ snapshot_publish ${MIRROR_SNAPSHOT_BASENAME} ${ISAR_MIRROR_PREFIX} ${DISTRO_SUITE}
+ snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-updates ${ISAR_MIRROR_PREFIX}-updates ${DISTRO_SUITE}
+ snapshot_publish ${MIRROR_SNAPSHOT_BASENAME}-security ${ISAR_MIRROR_PREFIX}-security ${DISTRO_SUITE}
fi
-}
+ if [ -e "${ISAR_FIRST_BUILD_DONE}" ] && [ "${REPRODUCIBLE_BUILD_ENABLED}" == "1" ]; then
+ cache_load_snapshot ${SNAPSHOT_BASENAME}_${MACHINE}
+ fi
+}
addtask cache_config after do_unpack before do_build
+do_cache_config[dirs] += "${CACHE_CONF_DIR}"
+do_cache_config[stamp-extra-info] = "${DISTRO}"
+do_cache_config[lockfiles] = "${TOPDIR}/apt-cache/isar.lock"
+
+
+# Return a string containing all packages required for setting up the mirror.
+# Implementing and running an own parser will also ensure considering bbappend files.
+def get_filter_list(d):
+ import sys
+ import bb.cooker, bb.cookerdata
+
+ #
+ # Todo: Move class into lib/oe
+ #
+ class DummyConfigParameters(bb.cookerdata.ConfigParameters):
+ """ Class for generating Dummy config parameters. Required for creating valid
+ cookerdata.ConfigParameters and setting these within cookerdata.CookerConfiguration
+ with cookerdata.CookerConfiguration.setConfigParameters().
+ """
+ def __init__(self, **options):
+ self.initial_options = options
+ super(DummyConfigParameters, self).__init__()
+
+ def parseCommandLine(self, argv=sys.argv):
+ class DummyOptions:
+ def __init__(self, initial_options):
+ for key, val in initial_options.items():
+ setattr(self, key, val)
+
+ return DummyOptions(self.initial_options), None
+
+
+ # Get collections
+ # A collection object stores information and methods about bbfiles as well as a list
+ # of bbfiles.
+ bbfile_config_priorities = []
+ collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
+ bbfiles, masked, searchdirs = collection.collect_bbfiles(d, d)
+
+ configparams = DummyConfigParameters()
+
+ # Parse of configuration data for a recipes' environment
+ configuration = bb.cookerdata.CookerConfiguration()
+ configuration.setConfigParameters(configparams)
+
+ # Get the databuilder, which is responsible for holding
+ # config related information.
+ databuilder = bb.cookerdata.CookerDataBuilder(configuration, False)
+ databuilder.parseBaseConfiguration()
+ parser = bb.cache.NoCache(databuilder)
+
+ prefetch_packages = ''
+
+ for bbfile in bbfiles:
+ appendfiles = collection.get_file_appends(bbfile)
+ bb.note('Loading data from {}'.format(bbfile))
+ data = parser.loadDataFull(bbfile, appendfiles)
+
+ buildchroot_preinstall = data.getVar('BUILDCHROOT_PREINSTALL', True) or ''
+ image_preinstall = data.getVar('IMAGE_PREINSTALL', True) or ''
+ debian_depends = data.getVar('DEBIAN_DEPENDS', True) or ''
+
+ # Runtime dependencies
+ buildchroot_preinstall = '|'.join(buildchroot_preinstall.split())
+ image_preinstall = '|'.join(image_preinstall.split())
+ debian_depends = debian_depends.replace(',', '|')
+
+ if buildchroot_preinstall:
+ prefetch_packages += buildchroot_preinstall + '|'
+ if image_preinstall:
+ prefetch_packages += image_preinstall + '|'
+ if debian_depends:
+ prefetch_packages += debian_depends + '|'
+
+ # Buildtime dependencies
+ workdir = data.getVar('WORKDIR', True) or ''
+ src_dir = data.getVar('SRC_DIR', True) or ''
+ control = os.path.join(workdir, src_dir, 'debian', 'control')
+
+ # Build-Depends
+ for dep in iter_deps(control, 'Build-Depends'):
+ bb.note('Found Build-Depends:%s in %s' % (dep, data.getVar('P')))
+ prefetch_packages += dep + '|'
+
+ # Build-Depends-Indep
+ for dep in iter_deps(control, 'Build-Depends-Indep'):
+ bb.note('Found Build-Depends-Indep:%s in %s' % (dep, data.getVar('P')))
+ prefetch_packages += dep + '|'
+
+ return prefetch_packages.strip('|')
+
+
+
+# Generator for iterating over package dependencies
+# Strings of type "pkg-name ([<>=] ${*:*})" are not supported,
+# they are returned as raw strings.
+def iter_deps(control, field):
+ import deb822
+
+ try:
+ f = open(control)
+ except:
+ return
+
+ for paragraph in deb822.Deb822.iter_paragraphs(f):
+ for item in paragraph.items():
+ if item[0] != field:
+ continue
+ archs = deb822.PkgRelation.parse_relations(item[1])
+ for name in archs:
+ for i in name:
+ yield(i['name'])
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 2/7] Added API class for apt cache.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 1/7] Implement support for setting up the local apt mirror and isar repository with aptly Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 3/7] Added apt-cache functionality for buildchroot Benedikt Niedermayr
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
This class implements a simple shell based API for accessing the underlying functionality of
aptly.
It covers following topics:
- mirror handling
- repo handling
- snapshot handling
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/classes/apt-cache.bbclass | 237 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 237 insertions(+)
create mode 100644 meta/classes/apt-cache.bbclass
diff --git a/meta/classes/apt-cache.bbclass b/meta/classes/apt-cache.bbclass
new file mode 100644
index 0000000..18ec80c
--- /dev/null
+++ b/meta/classes/apt-cache.bbclass
@@ -0,0 +1,237 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2018 Mixed-Mode GmbH
+#
+# This class implements common functionality for the isar apt cache.
+
+
+
+SNAPSHOT_BASENAME="repo-snapshot"
+MIRROR_SNAPSHOT_BASENAME="mirror-snapshot"
+
+ISAR_REPO = "${DEBDISTRONAME}-repo-local"
+
+ISAR_REPO_PREFIX = "${DEBDISTRONAME}"
+ISAR_MIRROR_PREFIX = "${DEBDISTRONAME}-mirror"
+
+MIRROR = "${DEBDISTRONAME}"
+MIRROR_SECURITY = "${DEBDISTRONAME}-security"
+MIRROR_UPDATES = "${DEBDISTRONAME}-updates"
+
+CACHE_DIR = "${TOPDIR}/apt-cache"
+CACHE_CONF_DIR = "${CACHE_DIR}/${DISTRO}/conf"
+CHROOT_CACHE_DIR = "/apt-cache"
+CFG_FILE = "${CACHE_CONF_DIR}/aptly.conf"
+ISAR_FIRST_BUILD_DONE = "${CACHE_CONF_DIR}/isar_first_build_done"
+
+# Reproducible build is turned on per default
+REPRODUCIBLE_BUILD_ENABLED ?= "1"
+
+
+# Setup a new empty debian mirror.
+# $1: The filter for delimiting packages
+# $2: Mirror
+# $3: Remote URL
+# $4: Suite
+mirror_create() {
+ aptly -config=${CFG_FILE} \
+ mirror create \
+ -architectures="${DISTRO_ARCH}" \
+ -filter="$1" \
+ -filter-with-deps \
+ $2 $3 $4
+}
+
+
+# Update a mirror, which may result in downloading new packages
+# $1: Mirror
+mirror_update() {
+ aptly -config=${CFG_FILE} mirror update $1
+}
+
+
+# Create a snapshot from a given mirror.
+# $1: Snapshot name.
+# $2: Mirror
+snapshot_create_from_mirror() {
+ aptly -config=${CFG_FILE} snapshot create $1 from mirror $2
+}
+
+
+# Create a snapshot from a given repository database
+# $1: Snapshot name.
+snapshot_create_from_repo() {
+ aptly -config=${CFG_FILE} \
+ snapshot create \
+ $1 from repo \
+ ${ISAR_REPO}
+}
+
+
+# Rename a snapshot
+# $1: Old name.
+# $2: New name.
+snapshot_rename() {
+ aptly -config=${CFG_FILE} snapshot rename $1 $2
+}
+
+# Delete a snapshot
+# $1: Snapshot name
+snapshot_delete() {
+ aptly -config=${CFG_FILE} \
+ snapshot drop \
+ $1 || bbnote "No snapshot to delete"
+}
+
+
+# Create the actual repository for a given snapshot.
+# $1: Snapshot name.
+# $2: Publish prefix
+# $3: Distribution
+snapshot_publish() {
+ aptly -config=${CFG_FILE} \
+ publish snapshot \
+ -architectures="${DISTRO_ARCH}" \
+ -distribution="$3" \
+ $1 \
+ $2
+}
+
+
+# Create a new repository database for Isar generated packages.
+# $1: Repository database name
+repo_db_create() {
+ aptly -config=${CFG_FILE} \
+ repo create \
+ -component="main" \
+ -distribution="${DEBDISTRONAME}" \
+ $1
+}
+
+# Create a new repository database, but
+# use a snapshot as source.
+# $1: Repository database name.
+# $2: Snapshot name
+repo_db_create_from_snapshot() {
+ aptly -config=${CFG_FILE} \
+ repo create \
+ -component="main" \
+ -distribution="${DEBDISTRONAME}" \
+ $1 \
+ from snapshot \
+ $2
+}
+
+
+# Drop a created repository database silenty.
+repo_db_drop() {
+ aptly -config=${CFG_FILE} \
+ repo drop -force \
+ ${ISAR_REPO} || bbnote "No db to drop"
+
+}
+
+
+# Add packages to the local repository database.
+# $1: Repository database name.
+# $*: Packages to add.
+repo_db_add_package() {
+ repo=$1
+ shift
+ aptly -config=${CFG_FILE} repo add -force-replace $repo $*
+}
+
+
+# Delete packages from the repository database.
+# $1: Repository database name.
+# $*: Packages to delete.
+repo_db_delete_package() {
+ repo=$1
+ shift
+ aptly -config=${CFG_FILE} repo remove $repo $*
+}
+
+
+# Publish repository consumed by apt. This function can only be called
+# for repositories
+# $1: Repository database name
+# $2: Path Prefix
+repo_db_publish() {
+ aptly -config=${CFG_FILE} \
+ publish repo \
+ -architectures="${DISTRO_ARCH}" \
+ $1 \
+ $2
+
+}
+
+
+# Drop an already published repository.
+# $1: Path prefix
+repo_db_publish_drop() {
+ aptly -config=${CFG_FILE} \
+ publish drop \
+ ${DEBDISTRONAME} \
+ $1 || bbnote "Nothing to publish"
+}
+
+
+# Switch already published repository to new repository. This
+# will change the content of the published repository. This function
+# can only be called on published repos.
+# $1: Prefix
+cache_update_repo() {
+ aptly -config=${CFG_FILE} \
+ publish update \
+ -force-overwrite \
+ ${DEBDISTRONAME} $1
+}
+
+
+# Add packages to the isar cache.
+# $1: Repository database name
+# $2: Repository prefix
+# $*: Packages to add
+cache_add_package() {
+ repo=$1
+ prefix=$2
+ shift 2
+ repo_db_add_package $repo $*
+ cache_update_repo $prefix
+}
+
+
+# Delete a package from the apt cache.
+# This function is used to be called within do_clean(all) task.
+# $1: Repository database name
+# $2: Repository prefix
+# $*: Packages to delete
+cache_delete_package() {
+ repo=$1
+ prefix=$2
+ shift 2
+ repo_db_delete_package $repo $*
+ cache_update_repo $prefix
+}
+
+
+# Create a snapshot, from current repository state
+cache_create_repo_snapshot() {
+ repo_db_publish_drop ${ISAR_REPO_PREFIX}
+ snapshot_delete ${SNAPSHOT_BASENAME}_${MACHINE}
+ snapshot_create_from_repo ${SNAPSHOT_BASENAME}_${MACHINE}
+ repo_db_publish ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+}
+
+# Load the last snapshot and publish it
+cache_load_snapshot() {
+ last_snapshot=$(aptly --config=${CFG_FILE} --raw -sort=time snapshot list | grep "$1" | tail -n 1)
+
+ if [ -z "$last_snapshot" ]; then
+ bbfatal "Requesting snapshot which has never been created. Repository may be in a inconsistent state. Totally new and clean build required."
+ fi
+ bbplain "Loading last snapshot: $last_snapshot"
+ repo_db_publish_drop ${ISAR_REPO_PREFIX}
+ repo_db_drop
+ repo_db_create_from_snapshot ${ISAR_REPO} ${SNAPSHOT_BASENAME}_${MACHINE}
+ repo_db_publish ${ISAR_REPO} ${ISAR_REPO_PREFIX}
+}
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 3/7] Added apt-cache functionality for buildchroot.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 1/7] Implement support for setting up the local apt mirror and isar repository with aptly Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 2/7] Added API class for apt cache Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 4/7] Added apt-cache functionality for image rootfs Benedikt Niedermayr
` (3 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
ESSENTIAL_INSTALL was added since multistrap won't pick up all required packages
when calculating the initial package list.
It seems that multiconfig silently drops some packages if they where not present at the repository.
The completion hook was added in order to support fetching from local repository within chroot environment.
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/recipes-devtools/buildchroot/buildchroot.bb | 36 +++++++++++++++++++---
.../buildchroot/files/configscript.sh | 1 +
.../buildchroot/files/multistrap.conf.in | 17 +++++-----
3 files changed, 41 insertions(+), 13 deletions(-)
diff --git a/meta/recipes-devtools/buildchroot/buildchroot.bb b/meta/recipes-devtools/buildchroot/buildchroot.bb
index 304c67e..91d8d4e 100644
--- a/meta/recipes-devtools/buildchroot/buildchroot.bb
+++ b/meta/recipes-devtools/buildchroot/buildchroot.bb
@@ -16,7 +16,15 @@ SRC_URI = "file://multistrap.conf.in \
file://build.sh"
PV = "1.0"
-BUILDCHROOT_PREINSTALL ?= "gcc \
+inherit apt-cache
+
+do_build[depends] = "isar-apt:do_cache_config"
+
+ESSENTIAL_INSTALL ?= "mawk \
+ "
+
+BUILDCHROOT_PREINSTALL ?= "${ESSENTIAL_INSTALL} \
+ gcc \
make \
build-essential \
debhelper \
@@ -47,17 +55,34 @@ do_build() {
cd ${TOPDIR}
WORKDIR_REL=${@ os.path.relpath(d.getVar("WORKDIR", True))}
+ # 1. Replace directory of apt caches for using within chroots and
+ # 2. set the trusted=yes option for local unsigned cache repository.
+ # 3. Reduce components to main since mirror only offers that,
+ # see Multi-Component Publishing section of aptly doc for avoiding this fix.
+ cat<<-__EOF__ > $WORKDIR_REL/hooks_multistrap/completion_sources
+ for section in ${DISTRO_MULTICONF_APTSOURCES} isar; do
+ sed -i -e 's|${CACHE_DIR}|${CHROOT_CACHE_DIR}|g' \
+ -e 's|\[|\[ trusted=yes |g' \
+ -e 's|${DISTRO_COMPONENTS}|main|g' \
+ ${BUILDCHROOT_DIR}/etc/apt/sources.list.d/multistrap-\$section.list
+ done
+
+ __EOF__
+ chmod 0755 $WORKDIR_REL/hooks_multistrap/completion_sources
+
# Adjust multistrap config
sed -e 's|##BUILDCHROOT_PREINSTALL##|${BUILDCHROOT_PREINSTALL}|g' \
-e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
-e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
- -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
- -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
+ -e 's|##DISTRO_APT_SOURCE##|copy:///${CACHE_DIR}/${DISTRO}/public/${ISAR_MIRROR_PREFIX}|g' \
+ -e 's|##DISTRO_APT_SOURCE_SEC##|copy:///${CACHE_DIR}/${DISTRO}/public/${ISAR_MIRROR_PREFIX}-security|g' \
-e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
-e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
-e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/configscript.sh|g' \
-e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
-e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
+ -e 's|##DEPLOY_DIR_APT##|copy:///${CACHE_DIR}/${DISTRO}/public/${ISAR_REPO_PREFIX}|g' \
+ -e 's|##ISAR_DISTRO_SUITE##|${DEBDISTRONAME}|g' \
"${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
[ ! -d ${BUILDCHROOT_DIR}/proc ] && install -d -m 555 ${BUILDCHROOT_DIR}/proc
@@ -95,11 +120,12 @@ python __anonymous() {
do_setup_mounts[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
do_setup_mounts() {
- sudo mount --bind ${DEPLOY_DIR_APT}/${DISTRO} ${BUILDCHROOT_DIR}/isar-apt
+ sudo mount --bind ${TOPDIR}/apt-cache ${BUILDCHROOT_DIR}${CHROOT_CACHE_DIR}
}
addtask setup_mounts after do_build
do_cleanup_mounts() {
- sudo umount ${BUILDCHROOT_DIR}/isar-apt 2>/dev/null || true
+ sudo umount ${BUILDCHROOT_DIR}${CHROOT_CACHE_DIR} 2>/dev/null || true
}
+do_build[dirs] += "${CACHE_DIR}/${DISTRO} ${BUILDCHROOT_DIR}/apt-cache"
\ No newline at end of file
diff --git a/meta/recipes-devtools/buildchroot/files/configscript.sh b/meta/recipes-devtools/buildchroot/files/configscript.sh
index 458c94b..9813c9a 100644
--- a/meta/recipes-devtools/buildchroot/files/configscript.sh
+++ b/meta/recipes-devtools/buildchroot/files/configscript.sh
@@ -44,4 +44,5 @@ mount -t devtmpfs -o mode=0755,nosuid devtmpfs /dev
#configuring packages
dpkg --configure -a
+apt-get update
umount /dev
diff --git a/meta/recipes-devtools/buildchroot/files/multistrap.conf.in b/meta/recipes-devtools/buildchroot/files/multistrap.conf.in
index 480a4b8..ffb7023 100644
--- a/meta/recipes-devtools/buildchroot/files/multistrap.conf.in
+++ b/meta/recipes-devtools/buildchroot/files/multistrap.conf.in
@@ -5,8 +5,8 @@
noauth=true
unpack=true
ignorenativearch=true
-bootstrap=##DISTRO_MULTICONF_BOOTSTRAP##
-aptsources=isar-apt ##DISTRO_MULTICONF_APTSOURCES##
+bootstrap=##DISTRO_MULTICONF_BOOTSTRAP## Isar
+aptsources=##DISTRO_MULTICONF_APTSOURCES## Isar
configscript=##CONFIG_SCRIPT##
setupscript=##SETUP_SCRIPT##
hookdir=##DIR_HOOKS##
@@ -19,19 +19,20 @@ packages=##BUILDCHROOT_PREINSTALL##
omitdebsrc=true
[updates]
-source=##DISTRO_APT_SOURCE##
-suite=##DISTRO_SUITE##-updates
+source=##DISTRO_APT_SOURCE##-updates
+suite=##DISTRO_SUITE##
components=##DISTRO_COMPONENTS##
omitdebsrc=true
[security]
source=##DISTRO_APT_SOURCE_SEC##
-suite=##DISTRO_SUITE##/updates
+suite=##DISTRO_SUITE##
components=##DISTRO_COMPONENTS##
omitdebsrc=true
-[isar-apt]
-source=file:///isar-apt
-suite=isar
+[Isar]
+packages=##BUILDCHROOT_PREINSTALL##
+source=##DEPLOY_DIR_APT##
+suite=##ISAR_DISTRO_SUITE##
components=main
omitdebsrc=true
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 4/7] Added apt-cache functionality for image rootfs.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
` (2 preceding siblings ...)
2018-02-08 10:40 ` [PATCH v2 3/7] Added apt-cache functionality for buildchroot Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 5/7] Added do_finalize_image task Benedikt Niedermayr
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
This patch does the same thing as the last patch does for buildchroot.
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
.../recipes-core/images/files/multistrap.conf.in | 8 +++----
meta-isar/recipes-core/images/isar-image-base.bb | 25 ++++++++++++++++++----
2 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/meta-isar/recipes-core/images/files/multistrap.conf.in b/meta-isar/recipes-core/images/files/multistrap.conf.in
index 432b6af..4bbf2e9 100644
--- a/meta-isar/recipes-core/images/files/multistrap.conf.in
+++ b/meta-isar/recipes-core/images/files/multistrap.conf.in
@@ -6,7 +6,7 @@ noauth=true
unpack=true
ignorenativearch=true
bootstrap=##DISTRO_MULTICONF_BOOTSTRAP## Isar
-aptsources=##DISTRO_MULTICONF_APTSOURCES##
+aptsources=##DISTRO_MULTICONF_APTSOURCES## Isar
configscript=##CONFIG_SCRIPT##
setupscript=##SETUP_SCRIPT##
hookdir=##DIR_HOOKS##
@@ -19,14 +19,14 @@ packages=##IMAGE_PREINSTALL##
omitdebsrc=true
[updates]
-source=##DISTRO_APT_SOURCE##
-suite=##DISTRO_SUITE##-updates
+source=##DISTRO_APT_SOURCE##-updates
+suite=##DISTRO_SUITE##
components=##DISTRO_COMPONENTS##
omitdebsrc=true
[security]
source=##DISTRO_APT_SOURCE_SEC##
-suite=##DISTRO_SUITE##/updates
+suite=##DISTRO_SUITE##
components=##DISTRO_COMPONENTS##
omitdebsrc=true
diff --git a/meta-isar/recipes-core/images/isar-image-base.bb b/meta-isar/recipes-core/images/isar-image-base.bb
index e359ac3..b342018 100644
--- a/meta-isar/recipes-core/images/isar-image-base.bb
+++ b/meta-isar/recipes-core/images/isar-image-base.bb
@@ -16,7 +16,7 @@ SRC_URI = "file://multistrap.conf.in \
PV = "1.0"
-inherit image
+inherit image apt-cache
DEPENDS += "${IMAGE_INSTALL}"
@@ -39,27 +39,43 @@ do_rootfs() {
cd ${TOPDIR}
WORKDIR_REL=${@ os.path.relpath(d.getVar("WORKDIR", True))}
+ # 1. Replace directory of apt caches for using within chroots and
+ # 2. set the trusted=yes option for local unsigned cache repository.
+ # 3. Reduce components to main since mirror only offers that,
+ # see Multi-Component Publishing section of aptly doc for avoiding this fix.
+ cat<<-__EOF__ > $WORKDIR_REL/hooks_multistrap/completion_sources
+ for section in ${DISTRO_MULTICONF_APTSOURCES} isar; do
+ sed -i -e 's|${CACHE_DIR}|${CHROOT_CACHE_DIR}|g' \
+ -e 's|\[|\[ trusted=yes |g' \
+ -e 's|${DISTRO_COMPONENTS}|main|g' \
+ ${IMAGE_ROOTFS}/etc/apt/sources.list.d/multistrap-\$section.list
+ done
+ __EOF__
+ chmod 0755 $WORKDIR_REL/hooks_multistrap/completion_sources
+
# Adjust multistrap config
sed -e 's|##IMAGE_PREINSTALL##|${IMAGE_PREINSTALL}|g' \
-e 's|##DISTRO_MULTICONF_BOOTSTRAP##|${DISTRO_MULTICONF_BOOTSTRAP}|g' \
-e 's|##DISTRO_MULTICONF_APTSOURCES##|${DISTRO_MULTICONF_APTSOURCES}|g' \
- -e 's|##DISTRO_APT_SOURCE##|${DISTRO_APT_SOURCE}|g' \
- -e 's|##DISTRO_APT_SOURCE_SEC##|${DISTRO_APT_SOURCE_SEC}|g' \
+ -e 's|##DISTRO_APT_SOURCE##|copy:///${CACHE_DIR}/${DISTRO}/public/${ISAR_MIRROR_PREFIX}|g' \
+ -e 's|##DISTRO_APT_SOURCE_SEC##|copy:///${CACHE_DIR}/${DISTRO}/public/${ISAR_MIRROR_PREFIX}-security|g' \
-e 's|##DISTRO_SUITE##|${DISTRO_SUITE}|g' \
-e 's|##DISTRO_COMPONENTS##|${DISTRO_COMPONENTS}|g' \
-e 's|##CONFIG_SCRIPT##|./'"$WORKDIR_REL"'/${DISTRO_CONFIG_SCRIPT}|g' \
-e 's|##SETUP_SCRIPT##|./'"$WORKDIR_REL"'/setup.sh|g' \
-e 's|##DIR_HOOKS##|./'"$WORKDIR_REL"'/hooks_multistrap|g' \
-e 's|##IMAGE_INSTALL##|${IMAGE_INSTALL}|g' \
- -e 's|##DEPLOY_DIR_APT##|copy:///${DEPLOY_DIR_APT}/${DISTRO}|g' \
+ -e 's|##DEPLOY_DIR_APT##|copy:///${CACHE_DIR}/${DISTRO}/public/isar|g' \
-e 's|##ISAR_DISTRO_SUITE##|${DEBDISTRONAME}|g' \
"${WORKDIR}/multistrap.conf.in" > "${WORKDIR}/multistrap.conf"
[ ! -d ${IMAGE_ROOTFS}/proc ] && sudo install -d -o 0 -g 0 -m 555 ${IMAGE_ROOTFS}/proc
sudo mount -t proc none ${IMAGE_ROOTFS}/proc
+ sudo mount --bind ${TOPDIR}/apt-cache ${IMAGE_ROOTFS}${CHROOT_CACHE_DIR}
_do_rootfs_cleanup() {
ret=$?
sudo umount ${IMAGE_ROOTFS}/proc 2>/dev/null || true
+ sudo umount ${IMAGE_ROOTFS}${CHROOT_CACHE_DIR} 2>/dev/null || true
(exit $ret) || bb_exit_handler
}
trap '_do_rootfs_cleanup' EXIT
@@ -74,3 +90,4 @@ do_rootfs() {
sudo rm "${IMAGE_ROOTFS}/${DISTRO_CONFIG_SCRIPT}"
_do_rootfs_cleanup
}
+do_rootfs[dirs] += "${IMAGE_ROOTFS}${CHROOT_CACHE_DIR}"
\ No newline at end of file
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 5/7] Added do_finalize_image task.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
` (3 preceding siblings ...)
2018-02-08 10:40 ` [PATCH v2 4/7] Added apt-cache functionality for image rootfs Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 6/7] Added support for installing isar packages to local isar repository Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 7/7] Introducing fetchall, unpackall and unpack_debian tasks Benedikt Niedermayr
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
This taks does two things:
- Create a snapshot for the current Isar repository after build has finished.
- Add remote repositories to apt source files.
Those files where not created, since multistrap only pulls packages from local mirrors.
Todos:
- multistrap-*.list files are hardcoded. This should be fixed in future, since different distributions
refer to different repositories.
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/classes/image.bbclass | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index cbd74b3..e1c7dd5 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -4,6 +4,7 @@
IMAGE_INSTALL ?= ""
IMAGE_TYPE ?= "ext4-img"
IMAGE_ROOTFS = "${WORKDIR}/rootfs"
+CACHE_STAGING_DIR = "${DEPLOY_DIR_APT}/${DISTRO}/staging"
def get_image_name(d, name_link):
S = d.getVar("IMAGE_ROOTFS", True)
@@ -16,7 +17,7 @@ def get_image_name(d, name_link):
KERNEL_IMAGE ?= "${@get_image_name(d, 'vmlinuz')}"
INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')}"
-inherit ${IMAGE_TYPE}
+inherit ${IMAGE_TYPE} apt-cache
do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
@@ -42,3 +43,23 @@ do_copy_boot_files() {
addtask copy_boot_files before do_build after do_rootfs
do_copy_boot_files[dirs] = "${DEPLOY_DIR_IMAGE}"
do_copy_boot_files[stamp-extra-info] = "${DISTRO}-${MACHINE}"
+
+
+do_finalize_image() {
+ sudo rm -f ${IMAGE_ROOTFS}/etc/apt/sources.list.d/multistrap-*.list
+
+ sudo sh -c 'echo "deb ${DISTRO_APT_SOURCE} ${DISTRO_SUITE} ${DISTRO_COMPONENTS}" > \
+ ${IMAGE_ROOTFS}/etc/apt/sources.list.d/multistrap-base.list'
+ sudo sh -c 'echo "deb ${DISTRO_APT_SOURCE} ${DISTRO_SUITE}-updates ${DISTRO_COMPONENTS}" > \
+ ${IMAGE_ROOTFS}/etc/apt/sources.list.d/multistrap-updates.list'
+ sudo sh -c 'echo "deb ${DISTRO_APT_SOURCE_SEC} ${DISTRO_SUITE}/updates ${DISTRO_COMPONENTS}" > \
+ ${IMAGE_ROOTFS}/etc/apt/sources.list.d/multistrap-security.list'
+
+ cache_create_repo_snapshot
+
+ if [ ! -e "${ISAR_FIRST_BUILD_DONE}" ]; then
+ touch ${ISAR_FIRST_BUILD_DONE}
+ fi
+}
+addtask do_finalize_image before do_build after do_copy_boot_files
+do_finalize_image[dirs] += "${DEPLOY_DIR_APT}/${DISTRO}/staging"
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 6/7] Added support for installing isar packages to local isar repository.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
` (4 preceding siblings ...)
2018-02-08 10:40 ` [PATCH v2 5/7] Added do_finalize_image task Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
2018-02-08 10:40 ` [PATCH v2 7/7] Introducing fetchall, unpackall and unpack_debian tasks Benedikt Niedermayr
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
Installs packages to the local Isar reposiory within the do_deploy_deb task.
The cache_add_package function does the following:
- Add the package to the repository database
- Update the published repository
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/classes/dpkg-base.bbclass | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index c5820fe..12daea5 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -1,6 +1,8 @@
# This software is a part of ISAR.
# Copyright (C) 2017 Siemens AG
+inherit apt-cache
+
# Add dependency from buildchroot creation
do_build[depends] = "buildchroot:do_setup_mounts"
@@ -24,10 +26,14 @@ dpkg_runbuild() {
do_build() {
mkdir -p ${BUILDROOT}
sudo mount --bind ${WORKDIR} ${BUILDROOT}
+ sudo mount --bind ${TOPDIR}/apt-cache ${BUILDCHROOT_DIR}${CHROOT_CACHE_DIR}
_do_build_cleanup() {
ret=$?
sudo umount ${BUILDROOT} 2>/dev/null || true
sudo rmdir ${BUILDROOT} 2>/dev/null || true
+
+ sudo umount ${BUILDCHROOT_DIR}${CHROOT_CACHE_DIR} 2>/dev/null || true
+ sudo rmdir ${BUILDCHROOT_DIR}${CHROOT_CACHE_DIR} 2>/dev/null || true
(exit $ret) || bb_exit_handler
}
trap '_do_build_cleanup' EXIT
@@ -35,13 +41,10 @@ do_build() {
_do_build_cleanup
}
-# Install package to Isar-apt
+
+# Install package to isar repository
do_deploy_deb() {
- reprepro -b ${DEPLOY_DIR_APT}/${DISTRO} \
- --dbdir ${DEPLOY_DIR_DB}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${WORKDIR}/*.deb
+ cache_add_package ${ISAR_REPO} ${ISAR_REPO_PREFIX} ${WORKDIR}/*.deb
}
addtask deploy_deb after do_build
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH v2 7/7] Introducing fetchall, unpackall and unpack_debian tasks.
2018-02-08 10:40 [PATCH v2 0/7] Local apt cache with aptly V2 Benedikt Niedermayr
` (5 preceding siblings ...)
2018-02-08 10:40 ` [PATCH v2 6/7] Added support for installing isar packages to local isar repository Benedikt Niedermayr
@ 2018-02-08 10:40 ` Benedikt Niedermayr
6 siblings, 0 replies; 8+ messages in thread
From: Benedikt Niedermayr @ 2018-02-08 10:40 UTC (permalink / raw)
To: isar-users; +Cc: Benedikt Niedermayr
- fetchall, unpackall:
These tasks offer the possibility for running fetch or unpack tasks for each recipe
in Isar.
After running bitbake "<image>:do_unpackall" Isar will fetch all sources and also unpack them into
related WORKDIRs.
In order to be able to apply the mirror prefetching mechanism, all debian control files must be available
for Isar (Build-Depends required). With all control files available do_cache_config() task should be able to collect
all Build-Depends.
- unpack_debian:
This task should be implemented within each recipe if needed. After the task has run the control file should be located at
${WORKDIR}/${SRC_DIR}/debian/control.
When fetching already debianized sources this task can be omitted.
Locating the control file at the correct position is needed for the dependency collector.
Signed-off-by: Benedikt Niedermayr <Benedikt.Niedermayr@mixed-mode.de>
---
meta/classes/base.bbclass | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 7d44f72..5b28707 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -94,6 +94,26 @@ python do_unpack() {
}
addtask unpack after do_fetch before do_build
+do_unpack[postfuncs] += "do_unpack_debian"
+
+do_unpack_debian() {
+ :
+}
+
+
+addtask fetchall after do_fetch
+do_fetchall[recrdeptask] = "do_fetchall do_fetch"
+do_fetchall[recideptask] = "do_${BB_DEFAULT_TASK}"
+do_fetchall() {
+ :
+}
+
+addtask unpackall after do_unpack
+do_unpackall[recrdeptask] = "do_unpackall do_unpack"
+do_unpackall[recideptask] = "do_${BB_DEFAULT_TASK}"
+do_unpackall() {
+ :
+}
addtask build
do_build[dirs] = "${TOPDIR}"
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread