From: Henning Schild <henning.schild@siemens.com>
To: isar-users@googlegroups.com
Cc: Vijai Kumar K <vijaikumar.kanagarajan@gmail.com>,
Baurzhan Ismagulov <ibr@radix50.net>,
Jan Kiszka <jan.kiszka@siemens.com>,
Henning Schild <henning.schild@siemens.com>
Subject: [PATCHv7 01/29] repository: new class to deal with repos
Date: Sat, 21 Mar 2020 09:31:20 +0100 [thread overview]
Message-ID: <20200321083148.26160-2-henning.schild@siemens.com> (raw)
In-Reply-To: <20200321083148.26160-1-henning.schild@siemens.com>
From: Henning Schild <henning.schild@siemens.com>
Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfcb..6fc57303 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository
compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
@@ -40,18 +41,15 @@ populate_base_apt() {
compare_pkg_md5sums "$package" "$base_apt_p" && continue
# md5sum differs, so remove the package from base-apt
- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
fi
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 5d2c161c..4702e52b 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository
DEPENDS ?= ""
@@ -142,31 +143,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}
-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}
addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 00000000..cf0fb5a2
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2020 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local keyfiles="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}
+Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
+Components: main
+EOF
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local file="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8b..7c891fa0 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${KEYFILES}"
}
python do_cache_config() {
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index 3cf7ea55..00000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}
-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index 64717364..00000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}
-Architectures: i386 armhf arm64 amd64 mipsel riscv64 source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753d..c53b01f7 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@
# This software is a part of ISAR.
# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2020 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}"
}
addtask cache_config after do_unpack before do_build
--
2.24.1
next prev parent reply other threads:[~2020-03-21 8:31 UTC|newest]
Thread overview: 41+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-21 8:31 [PATCHv7 00/29] base-apt-rework Henning Schild
2020-03-21 8:31 ` Henning Schild [this message]
2020-03-21 8:31 ` [PATCHv7 02/29] dpkg-base: add download caching of apt:// downloads Henning Schild
2020-03-21 8:31 ` [PATCHv7 03/29] meta: dpkg-base: convert "exit" into "return 0" Henning Schild
2020-03-21 8:31 ` [PATCHv7 04/29] base-apt: change the sources.list to also offer deb-src Henning Schild
2020-03-21 8:31 ` [PATCHv7 05/29] base-apt: add deb-src packages as well Henning Schild
2020-03-21 8:31 ` [PATCHv7 06/29] base-apt: do not skip gpg check when it is signed Henning Schild
2020-03-21 8:31 ` [PATCHv7 07/29] ci: conf: add "hello" to the sample config and every build Henning Schild
2020-03-21 8:31 ` [PATCHv7 08/29] meta: split all apt-get invocations into download and execution Henning Schild
2020-03-21 8:31 ` [PATCHv7 09/29] meta: create DL_DIR support for all apt-get downloaded .debs Henning Schild
2020-03-21 8:31 ` [PATCHv7 10/29] meta: import DL_DIR debs before apt-get download steps Henning Schild
2020-03-21 8:31 ` [PATCHv7 11/29] meta: include DL_DIR deb import/export into bootstrap Henning Schild
2020-03-21 8:31 ` [PATCHv7 12/29] base-apt: populate from DEBDIR as well Henning Schild
2020-03-21 8:31 ` [PATCHv7 13/29] base-apt: drop the "apt_cache" feature Henning Schild
2020-03-21 8:31 ` [PATCHv7 14/29] base-apt: do not copy debs directly out of rootfss anymore Henning Schild
2020-03-21 8:31 ` [PATCHv7 15/29] base-apt: rework base-apt population Henning Schild
2020-03-21 8:31 ` [PATCHv7 16/29] base-apt: move class "base-apt-helper" into only user Henning Schild
2020-03-21 8:31 ` [PATCHv7 17/29] CI: include "isar-disable-apt-cache" into all CI images Henning Schild
2020-03-21 8:31 ` [PATCHv7 18/29] CI: include "cowsay" into default build to test dpkg-gbp Henning Schild
2020-03-21 8:31 ` [PATCHv7 19/29] CI: set BB_NO_NETWORK for cached rebuild Henning Schild
2020-03-21 8:31 ` [PATCHv7 20/29] meta: repository: implement repo_contains_package and use it in base_apt Henning Schild
2020-03-21 8:31 ` [PATCHv7 21/29] meta: repository: simplify the matching of packages Henning Schild
2020-03-21 8:31 ` [PATCHv7 22/29] meta: deb-dl-dir: do not cache debs from isar-apt Henning Schild
2020-03-21 8:31 ` [PATCHv7 23/29] base-apt: pull base-apt population to the front of the build chain Henning Schild
2020-03-21 8:31 ` [PATCHv7 24/29] meta: base-apt: remove isar-apt check from population Henning Schild
2020-03-21 8:31 ` [PATCHv7 25/29] meta: deb-dl-dir: only export newly downloaded files Henning Schild
2020-03-21 8:31 ` [PATCHv7 26/29] CI: test a custom kernel build in the base-apt offline rebuild Henning Schild
2020-03-21 8:31 ` [PATCHv7 27/29] meta/deb-dl-dir: make subshell verbose and fail on error Henning Schild
2020-03-21 8:31 ` [PATCHv7 28/29] meta: deb-dl-dir: fix "command not found" error when excluding pkgs Henning Schild
2020-03-21 8:31 ` [PATCHv7 29/29] RECIPE-API-CHANGELOG: add a few bits on the base-apt caching changes Henning Schild
2020-03-21 8:41 ` [PATCHv7 00/29] base-apt-rework Henning Schild
2020-03-23 10:56 ` Baurzhan Ismagulov
2020-03-23 11:01 ` Jan Kiszka
2020-03-23 12:22 ` Baurzhan Ismagulov
2020-03-23 15:25 ` Henning Schild
2020-03-24 10:49 ` Baurzhan Ismagulov
2020-03-25 10:35 ` Henning Schild
2020-03-25 11:13 ` Jan Kiszka
2020-03-25 11:25 ` Henning Schild
2020-03-25 11:14 ` Baurzhan Ismagulov
2020-03-26 12:53 ` Henning Schild
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200321083148.26160-2-henning.schild@siemens.com \
--to=henning.schild@siemens.com \
--cc=ibr@radix50.net \
--cc=isar-users@googlegroups.com \
--cc=jan.kiszka@siemens.com \
--cc=vijaikumar.kanagarajan@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox