* [PATCHv2 01/25] repository: new class to deal with repos
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads Henning Schild
` (24 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfc..1f08c70 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository
compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
@@ -40,18 +41,15 @@ populate_base_apt() {
compare_pkg_md5sums "$package" "$base_apt_p" && continue
# md5sum differs, so remove the package from base-apt
- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
fi
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index af9c984..f38168c 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository
DEPENDS ?= ""
@@ -140,31 +141,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}
-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}
addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 0000000..880ddef
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2019 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local keyfiles="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}
+Architectures: i386 armhf arm64 amd64 mipsel source
+Components: main
+EOF
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local file="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8..c9a7dad 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${KEYFILES}"
}
python do_cache_config() {
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index fd7de0b..0000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index a24a700..0000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753..0227c1b 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@
# This software is a part of ISAR.
# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}"
}
addtask cache_config after do_unpack before do_build
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
2019-12-16 11:49 ` [PATCHv2 01/25] repository: new class to deal with repos Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2020-01-09 11:27 ` vijai kumar
2019-12-16 11:49 ` [PATCHv2 03/25] meta: dpkg-base: convert "exit" into "return 0" Henning Schild
` (23 subsequent siblings)
25 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Store the results of "apt-get source" in the DL_DIR. This means we
really only need to fetch once on incremental rebuilds or if multiple
recipes use the same SRC_URI.
We can also later collect the results and put them in a repository.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
meta/conf/bitbake.conf | 1 +
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index f38168c..7a97d91 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -63,14 +63,30 @@ do_apt_fetch() {
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
- sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
- sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}'
+ for uri in "${SRC_APT}"; do
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ done
+
dpkg_undo_mounts
}
addtask apt_fetch after do_unpack before do_patch
do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
+addtask cleanall_apt before do_cleanall
+do_cleanall_apt[nostamp] = "1"
+do_cleanall_apt() {
+ if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
+ exit
+ fi
+ for uri in "${SRC_APT}"; do
+ rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
+ done
+}
+
def get_package_srcdir(d):
s = os.path.abspath(d.getVar("S", True))
workdir = os.path.abspath(d.getVar("WORKDIR", True))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 0eb3b54..ab608da 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads
2019-12-16 11:49 ` [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads Henning Schild
@ 2020-01-09 11:27 ` vijai kumar
2020-01-10 9:51 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: vijai kumar @ 2020-01-09 11:27 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 2933 bytes --]
On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild wrote:
>
> From: Henning Schild <hennin...@siemens.com <javascript:>>
>
> Store the results of "apt-get source" in the DL_DIR. This means we
> really only need to fetch once on incremental rebuilds or if multiple
> recipes use the same SRC_URI.
> We can also later collect the results and put them in a repository.
>
> Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> ---
> meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
> meta/conf/bitbake.conf | 1 +
> 2 files changed, 19 insertions(+), 2 deletions(-)
>
> diff --git a/meta/classes/dpkg-base.bbclass
> b/meta/classes/dpkg-base.bbclass
> index f38168c..7a97d91 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -63,14 +63,30 @@ do_apt_fetch() {
> -o Dir::Etc::SourceParts="-" \
> -o APT::Get::List-Cleanup="0"
>
> - sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
> - sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}'
> + for uri in "${SRC_APT}"; do
> + sudo -E chroot --userspec=$( id -u ):$( id -g )
> ${BUILDCHROOT_DIR} \
> + sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd
> /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source
> source "$2"' my_script "${DISTRO}" "${uri}"
> + sudo -E chroot --userspec=$( id -u ):$( id -g )
> ${BUILDCHROOT_DIR} \
> + sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} &&
> apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}"
>
Hi Henning,
Thank you for the base-apt rework.
BTW. What does my_script do here?
Thanks,
Vijai Kumar K
+ done
> +
> dpkg_undo_mounts
> }
>
> addtask apt_fetch after do_unpack before do_patch
> do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
>
> +addtask cleanall_apt before do_cleanall
> +do_cleanall_apt[nostamp] = "1"
> +do_cleanall_apt() {
> + if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
> + exit
> + fi
> + for uri in "${SRC_APT}"; do
> + rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
> + done
> +}
> +
> def get_package_srcdir(d):
> s = os.path.abspath(d.getVar("S", True))
> workdir = os.path.abspath(d.getVar("WORKDIR", True))
> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> index 0eb3b54..ab608da 100644
> --- a/meta/conf/bitbake.conf
> +++ b/meta/conf/bitbake.conf
> @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
> FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
> FILESEXTRAPATHS ?= "__default:"
> GITDIR = "${DL_DIR}/git"
> +DEBSRCDIR = "${DL_DIR}/deb-src"
> P = "${PN}-${PV}"
> PF = "${PN}-${PV}-${PR}"
> PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0]
> or 'defaultpkgname'}"
> --
> 2.24.1
>
>
[-- Attachment #1.2: Type: text/html, Size: 4405 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads
2020-01-09 11:27 ` vijai kumar
@ 2020-01-10 9:51 ` Henning Schild
2020-01-10 9:58 ` vijai kumar
0 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2020-01-10 9:51 UTC (permalink / raw)
To: vijai kumar; +Cc: isar-users
Am Thu, 9 Jan 2020 03:27:25 -0800
schrieb vijai kumar <vijaikumar.kanagarajan@gmail.com>:
> On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild
> wrote:
> >
> > From: Henning Schild <hennin...@siemens.com <javascript:>>
> >
> > Store the results of "apt-get source" in the DL_DIR. This means we
> > really only need to fetch once on incremental rebuilds or if
> > multiple recipes use the same SRC_URI.
> > We can also later collect the results and put them in a repository.
> >
> > Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> > ---
> > meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
> > meta/conf/bitbake.conf | 1 +
> > 2 files changed, 19 insertions(+), 2 deletions(-)
> >
> > diff --git a/meta/classes/dpkg-base.bbclass
> > b/meta/classes/dpkg-base.bbclass
> > index f38168c..7a97d91 100644
> > --- a/meta/classes/dpkg-base.bbclass
> > +++ b/meta/classes/dpkg-base.bbclass
> > @@ -63,14 +63,30 @@ do_apt_fetch() {
> > -o Dir::Etc::SourceParts="-" \
> > -o APT::Get::List-Cleanup="0"
> >
> > - sudo -E chroot --userspec=$( id -u ):$( id -g )
> > ${BUILDCHROOT_DIR} \
> > - sh -c 'cd ${PP} && apt-get -y --only-source source
> > ${SRC_APT}'
> > + for uri in "${SRC_APT}"; do
> > + sudo -E chroot --userspec=$( id -u ):$( id -g )
> > ${BUILDCHROOT_DIR} \
> > + sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd
> > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only
> > --only-source source "$2"' my_script "${DISTRO}" "${uri}"
> > + sudo -E chroot --userspec=$( id -u ):$( id -g )
> > ${BUILDCHROOT_DIR} \
> > + sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd
> > ${PP} && apt-get -y --only-source source "$2"' my_script
> > "${DISTRO}" "${uri}"
>
> Hi Henning,
>
> Thank you for the base-apt rework.
> BTW. What does my_script do here?
It fills $0 of the embedded shell script with a "program name".
Henning
> Thanks,
> Vijai Kumar K
>
> + done
> > +
> > dpkg_undo_mounts
> > }
> >
> > addtask apt_fetch after do_unpack before do_patch
> > do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
> >
> > +addtask cleanall_apt before do_cleanall
> > +do_cleanall_apt[nostamp] = "1"
> > +do_cleanall_apt() {
> > + if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
> > + exit
> > + fi
> > + for uri in "${SRC_APT}"; do
> > + rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
> > + done
> > +}
> > +
> > def get_package_srcdir(d):
> > s = os.path.abspath(d.getVar("S", True))
> > workdir = os.path.abspath(d.getVar("WORKDIR", True))
> > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > index 0eb3b54..ab608da 100644
> > --- a/meta/conf/bitbake.conf
> > +++ b/meta/conf/bitbake.conf
> > @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
> > FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
> > FILESEXTRAPATHS ?= "__default:"
> > GITDIR = "${DL_DIR}/git"
> > +DEBSRCDIR = "${DL_DIR}/deb-src"
> > P = "${PN}-${PV}"
> > PF = "${PN}-${PV}-${PR}"
> > PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE',
> > False),d)[0] or 'defaultpkgname'}"
> > --
> > 2.24.1
> >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads
2020-01-10 9:51 ` Henning Schild
@ 2020-01-10 9:58 ` vijai kumar
0 siblings, 0 replies; 68+ messages in thread
From: vijai kumar @ 2020-01-10 9:58 UTC (permalink / raw)
To: Henning Schild; +Cc: isar-users
[-- Attachment #1: Type: text/plain, Size: 3539 bytes --]
On Fri 10 Jan, 2020, 3:21 PM Henning Schild, <henning.schild@siemens.com>
wrote:
> Am Thu, 9 Jan 2020 03:27:25 -0800
> schrieb vijai kumar <vijaikumar.kanagarajan@gmail.com>:
>
> > On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild
> > wrote:
> > >
> > > From: Henning Schild <hennin...@siemens.com <javascript:>>
> > >
> > > Store the results of "apt-get source" in the DL_DIR. This means we
> > > really only need to fetch once on incremental rebuilds or if
> > > multiple recipes use the same SRC_URI.
> > > We can also later collect the results and put them in a repository.
> > >
> > > Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> > > ---
> > > meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
> > > meta/conf/bitbake.conf | 1 +
> > > 2 files changed, 19 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/meta/classes/dpkg-base.bbclass
> > > b/meta/classes/dpkg-base.bbclass
> > > index f38168c..7a97d91 100644
> > > --- a/meta/classes/dpkg-base.bbclass
> > > +++ b/meta/classes/dpkg-base.bbclass
> > > @@ -63,14 +63,30 @@ do_apt_fetch() {
> > > -o Dir::Etc::SourceParts="-" \
> > > -o APT::Get::List-Cleanup="0"
> > >
> > > - sudo -E chroot --userspec=$( id -u ):$( id -g )
> > > ${BUILDCHROOT_DIR} \
> > > - sh -c 'cd ${PP} && apt-get -y --only-source source
> > > ${SRC_APT}'
> > > + for uri in "${SRC_APT}"; do
> > > + sudo -E chroot --userspec=$( id -u ):$( id -g )
> > > ${BUILDCHROOT_DIR} \
> > > + sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd
> > > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only
> > > --only-source source "$2"' my_script "${DISTRO}" "${uri}"
> > > + sudo -E chroot --userspec=$( id -u ):$( id -g )
> > > ${BUILDCHROOT_DIR} \
> > > + sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd
> > > ${PP} && apt-get -y --only-source source "$2"' my_script
> > > "${DISTRO}" "${uri}"
> >
> > Hi Henning,
> >
> > Thank you for the base-apt rework.
> > BTW. What does my_script do here?
>
> It fills $0 of the embedded shell script with a "program name".
>
> Henning
>
Ah. I see. I should have known :) my bad.
Thanks,
Vijai Kumar K
> Thanks,
> > Vijai Kumar K
> >
> > + done
> > > +
> > > dpkg_undo_mounts
> > > }
> > >
> > > addtask apt_fetch after do_unpack before do_patch
> > > do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
> > >
> > > +addtask cleanall_apt before do_cleanall
> > > +do_cleanall_apt[nostamp] = "1"
> > > +do_cleanall_apt() {
> > > + if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
> > > + exit
> > > + fi
> > > + for uri in "${SRC_APT}"; do
> > > + rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
> > > + done
> > > +}
> > > +
> > > def get_package_srcdir(d):
> > > s = os.path.abspath(d.getVar("S", True))
> > > workdir = os.path.abspath(d.getVar("WORKDIR", True))
> > > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > > index 0eb3b54..ab608da 100644
> > > --- a/meta/conf/bitbake.conf
> > > +++ b/meta/conf/bitbake.conf
> > > @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
> > > FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
> > > FILESEXTRAPATHS ?= "__default:"
> > > GITDIR = "${DL_DIR}/git"
> > > +DEBSRCDIR = "${DL_DIR}/deb-src"
> > > P = "${PN}-${PV}"
> > > PF = "${PN}-${PV}-${PR}"
> > > PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE',
> > > False),d)[0] or 'defaultpkgname'}"
> > > --
> > > 2.24.1
> > >
> > >
> >
>
>
[-- Attachment #2: Type: text/html, Size: 5786 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread
* [PATCHv2 03/25] meta: dpkg-base: convert "exit" into "return 0"
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
2019-12-16 11:49 ` [PATCHv2 01/25] repository: new class to deal with repos Henning Schild
2019-12-16 11:49 ` [PATCHv2 02/25] dpkg-base: add download caching of apt:// downloads Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 04/25] base-apt: change the sources.list to also offer deb-src Henning Schild
` (22 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Bitbake seems to execute all the functions in subshells, but in fact we
mean to return here so correct the style.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-base.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 7a97d91..aa11660 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -53,7 +53,7 @@ SRC_APT ?= ""
do_apt_fetch() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
rm -rf ${S}
dpkg_do_mounts
@@ -80,7 +80,7 @@ addtask cleanall_apt before do_cleanall
do_cleanall_apt[nostamp] = "1"
do_cleanall_apt() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
for uri in "${SRC_APT}"; do
rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 04/25] base-apt: change the sources.list to also offer deb-src
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (2 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 03/25] meta: dpkg-base: convert "exit" into "return 0" Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 05/25] base-apt: add deb-src packages as well Henning Schild
` (21 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The repository is ready for it already. For the moment there will be
nothing there.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-core/isar-bootstrap/files/base-apt-sources | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
index ddc4509..2a4ab2c 100644
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
@@ -1 +1,2 @@
deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
+deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 05/25] base-apt: add deb-src packages as well
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (3 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 04/25] base-apt: change the sources.list to also offer deb-src Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 06/25] base-apt: do not skip gpg check when it is signed Henning Schild
` (20 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
So far the deb-src packages downloaded via "apt://" where not cached.
This commit fixes that and includes those packages into the cache.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 7 +++++++
meta/classes/image-cache-extension.bbclass | 4 ++++
meta/classes/repository.bbclass | 14 ++++++++++++++
3 files changed, 25 insertions(+)
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 1f08c70..db25adc 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -52,4 +52,11 @@ populate_base_apt() {
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
+
+ find $search_dir -name '*.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 30db549..f8b9ba3 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -32,5 +32,9 @@ do_cache_base_repo() {
if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+
+ if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
+ fi
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 880ddef..dc4c1dc 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -35,6 +35,20 @@ EOF
fi
}
+repo_add_srcpackage() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ includedsc "${codename}" \
+ "$@"
+}
+
repo_add_packages() {
local dir="$1"
local dbdir="$2"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 06/25] base-apt: do not skip gpg check when it is signed
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (4 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 05/25] base-apt: add deb-src packages as well Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 07/25] ci: conf: add "hello" to the sample config and every build Henning Schild
` (19 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We used to have "trusted=yes" hardcoded even for a signed base-apt. Make
that flag depend on whether it is signed and generate that file with two
echos, it is simple enough.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
.../isar-bootstrap/files/base-apt-sources | 2 --
.../isar-bootstrap/isar-bootstrap.inc | 16 ++++++++--------
2 files changed, 8 insertions(+), 10 deletions(-)
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources
diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
deleted file mode 100644
index 2a4ab2c..0000000
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ /dev/null
@@ -1,2 +0,0 @@
-deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
-deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5dd571e..cfcf7b8 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -12,8 +12,7 @@ SRC_URI = " \
file://isar-apt.conf \
file://isar-apt-fallback.conf \
file://locale \
- file://chroot-setup.sh \
- file://base-apt-sources"
+ file://chroot-setup.sh"
PV = "1.0"
DEBOOTSTRAP ?= "qemu-debootstrap"
@@ -21,7 +20,6 @@ ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-BASEAPTSRCS = "${WORKDIR}/base-apt-sources"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
@@ -266,13 +264,15 @@ isar_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sed -e "s#{BASE_DISTRO}#"${BASE_DISTRO}"#g" \
- -e "s#{BASE_DISTRO_CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- -i ${BASEAPTSRCS}
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
mkdir -p ${ROOTFSDIR}/base-apt
sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- install -v -m644 "${BASEAPTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
else
install -v -m644 "${APTSRCS}" \
"${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 07/25] ci: conf: add "hello" to the sample config and every build
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (5 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 06/25] base-apt: do not skip gpg check when it is signed Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 08/25] meta: split all apt-get invocations into download and execution Henning Schild
` (18 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Now that base-apt supports deb-src we can include hello in a standard
build.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 280e007..0e55da1 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,11 +166,9 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
## "cache_base_repo" relies on the cache ...
#IMAGE_INSTALL += "isar-disable-apt-cache"
-## uses apt:// SRC_URI, which does not work with the cache
-#IMAGE_INSTALL += "hello"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7cd5a2f..f50c9e4 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -169,7 +169,6 @@ if [ -n "$REPRO_BUILD" ]; then
fi
sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-sed -i -e 's/#IMAGE_INSTALL += "hello"/IMAGE_INSTALL += "hello"/g' conf/local.conf
echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 08/25] meta: split all apt-get invocations into download and execution
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (6 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 07/25] ci: conf: add "hello" to the sample config and every build Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs Henning Schild
` (17 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52..2afe9db 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 180a3a8..b26c645 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,7 +7,10 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216d..b8531cb 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -30,6 +30,11 @@ do_install_imager_deps() {
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+
+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
index 002085b..0f11d78 100644
--- a/meta/recipes-devtools/buildchroot/files/deps.sh
+++ b/meta/recipes-devtools/buildchroot/files/deps.sh
@@ -12,27 +12,34 @@ source /isar/common.sh
# 2) we add -y to go non-interactive
# 3) downgrades shall be allowed in case a package recipe was changed
install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends \
- -y --allow-downgrades"
+ -y --allow-downgrades $3"
-# Make sure that we have latest isar-apt content.
-# Options meaning:
-# Dir::Etc::SourceList - specifies which source to be used
-# Dir::Etc::SourceParts - disables looking for the other sources
-# APT::Get::List-Cleanup - do not erase obsolete packages list for
-# upstream in '/var/lib/apt/lists'
-apt-get update \
- -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
- -o Dir::Etc::SourceParts="-" \
- -o APT::Get::List-Cleanup="0"
+if [ "$3" != "-download-only" ]; then
+ # Make sure that we have latest isar-apt content.
+ # Options meaning:
+ # Dir::Etc::SourceList - specifies which source to be used
+ # Dir::Etc::SourceParts - disables looking for the other sources
+ # APT::Get::List-Cleanup - do not erase obsolete packages list for
+ # upstream in '/var/lib/apt/lists'
+ apt-get update \
+ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::SourceParts="-" \
+ -o APT::Get::List-Cleanup="0"
+fi
# Do not set an architecture when building only 'all' (generic) packages.
# This can avoid unneeded cross-build issues.
if ! grep "^Architecture:" debian/control | grep -qv "all"; then
- set_arch=""
+ set_arch=""
fi
# Install all build deps
-mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
+if [ "$3" = "--download-only" ]; then
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (7 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 08/25] meta: split all apt-get invocations into download and execution Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2020-01-25 14:06 ` vijai kumar
2019-12-16 11:49 ` [PATCHv2 10/25] meta: import DL_DIR debs before apt-get download steps Henning Schild
` (16 subsequent siblings)
25 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Hook in between the "apt-get --download-only"s and the "apt-get"s and
copy out all the debs for later.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
meta/classes/dpkg-base.bbclass | 1 +
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 12 ++++++++++--
meta/conf/bitbake.conf | 1 +
8 files changed, 31 insertions(+), 3 deletions(-)
create mode 100644 meta/classes/deb-dl-dir.bbclass
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
new file mode 100644
index 0000000..e41e981
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@
+# This software is a part of ISAR.
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+deb_dl_dir_export() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ mkdir -p "${pc}"
+ flock "${pc}".lock -c ' \
+ sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
+ -exec cp -f '{}' "${pc}" \;
+ sudo chown -R $(id -u):$(id -g) "${pc}"
+ '
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index aa11660..fd0b07f 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
inherit debianize
inherit terminal
inherit repository
+inherit deb-dl-dir
DEPENDS ?= ""
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 2afe9db..7e908eb 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index b26c645..7e2c86d 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
E="${@ isar_export_proxies(d)}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 3c0758f..0f0d0ca 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
j.split()[0].split(".")[0],
j.split()[0]))))
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download() {
sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index b8531cb..ec67d94 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'
+ deb_dl_dir_export ${BUILDCHROOT_DIR}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index b4dbfa3..a6b6110 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,6 +1,8 @@
# This software is a part of ISAR.
# Copyright (c) Siemens AG, 2019
+inherit deb-dl-dir
+
ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
ROOTFS_PACKAGES ?= ""
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
-ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
+
+ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] = "5"
+rootfs_export_package_cache() {
+ deb_dl_dir_export ${ROOTFSDIR}
+}
ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index ab608da..d86c5b9 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBDIR = "${DL_DIR}/deb"
DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs
2019-12-16 11:49 ` [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs Henning Schild
@ 2020-01-25 14:06 ` vijai kumar
2020-01-28 14:43 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: vijai kumar @ 2020-01-25 14:06 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 6724 bytes --]
On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild wrote:
>
> From: Henning Schild <hennin...@siemens.com <javascript:>>
>
> Hook in between the "apt-get --download-only"s and the "apt-get"s and
> copy out all the debs for later.
>
> Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> ---
> meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
> meta/classes/dpkg-base.bbclass | 1 +
> meta/classes/dpkg-gbp.bbclass | 1 +
> meta/classes/dpkg.bbclass | 1 +
> meta/classes/image-locales-extension.bbclass | 2 +-
> meta/classes/image-tools-extension.bbclass | 1 +
> meta/classes/rootfs.bbclass | 12 ++++++++++--
> meta/conf/bitbake.conf | 1 +
> 8 files changed, 31 insertions(+), 3 deletions(-)
> create mode 100644 meta/classes/deb-dl-dir.bbclass
>
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass
> new file mode 100644
> index 0000000..e41e981
> --- /dev/null
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -0,0 +1,15 @@
> +# This software is a part of ISAR.
> +# Copyright (C) 2019 Siemens AG
> +#
> +# SPDX-License-Identifier: MIT
> +
> +deb_dl_dir_export() {
> + export pc="${DEBDIR}/${DISTRO}"
> + export rootfs="${1}"
> + mkdir -p "${pc}"
> + flock "${pc}".lock -c ' \
> + sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname
> '*\.deb' \
> + -exec cp -f '{}' "${pc}" \;
> + sudo chown -R $(id -u):$(id -g) "${pc}"
> + '
> +}
> diff --git a/meta/classes/dpkg-base.bbclass
> b/meta/classes/dpkg-base.bbclass
> index aa11660..fd0b07f 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -8,6 +8,7 @@ inherit buildchroot
> inherit debianize
> inherit terminal
> inherit repository
> +inherit deb-dl-dir
>
> DEPENDS ?= ""
>
> diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
> index 2afe9db..7e908eb 100644
> --- a/meta/classes/dpkg-gbp.bbclass
> +++ b/meta/classes/dpkg-gbp.bbclass
> @@ -15,6 +15,7 @@ do_install_builddeps_append() {
> sudo -E chroot ${BUILDCHROOT_DIR} \
> apt-get install -y -o Debug::pkgProblemResolver=yes \
> --no-install-recommends --download-only
> ${GBP_DEPENDS}
> + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> sudo -E chroot ${BUILDCHROOT_DIR} \
> apt-get install -y -o Debug::pkgProblemResolver=yes \
> --no-install-recommends ${GBP_DEPENDS}
> diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> index b26c645..7e2c86d 100644
> --- a/meta/classes/dpkg.bbclass
> +++ b/meta/classes/dpkg.bbclass
> @@ -9,6 +9,7 @@ do_install_builddeps() {
While testing this series in a downstream project, I do not see a part of
KBUILD_DEPENDS cached.
We don't have the "linux-custom recipe rework" changes by Cedric there. The
old Linux recipe overrides
this task with its own variant.
I have added deb_dl_* there as well to fix the issue.
This might become a problem when more such recipes pop-up. Unlikely, but
possible.
Recipe authors should know they have to take care of this if they decide to
override
do_install_builddeps.
Thanks,
Vijai Kumar K
E="${@ isar_export_proxies(d)}"
> sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
> ${PP}/${PPS} ${DISTRO_ARCH} --download-only
> + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
> ${PP}/${PPS} ${DISTRO_ARCH}
> dpkg_undo_mounts
> diff --git a/meta/classes/image-locales-extension.bbclass
> b/meta/classes/image-locales-extension.bbclass
> index 3c0758f..0f0d0ca 100644
> --- a/meta/classes/image-locales-extension.bbclass
> +++ b/meta/classes/image-locales-extension.bbclass
> @@ -25,7 +25,7 @@ def get_nopurge(d):
> j.split()[0].split(".")[0],
> j.split()[0]))))
>
> -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN +=
> "image_install_localepurge_download"
> +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT +=
> "image_install_localepurge_download"
> image_install_localepurge_download[weight] = "40"
> image_install_localepurge_download() {
> sudo -E chroot '${ROOTFSDIR}' \
> diff --git a/meta/classes/image-tools-extension.bbclass
> b/meta/classes/image-tools-extension.bbclass
> index b8531cb..ec67d94 100644
> --- a/meta/classes/image-tools-extension.bbclass
> +++ b/meta/classes/image-tools-extension.bbclass
> @@ -34,6 +34,7 @@ do_install_imager_deps() {
> --allow-unauthenticated --allow-downgrades --download-only
> install \
> ${IMAGER_INSTALL}'
>
> + deb_dl_dir_export ${BUILDCHROOT_DIR}
> sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
> apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends
> -y \
> --allow-unauthenticated --allow-downgrades install \
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index b4dbfa3..a6b6110 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -1,6 +1,8 @@
> # This software is a part of ISAR.
> # Copyright (c) Siemens AG, 2019
>
> +inherit deb-dl-dir
> +
> ROOTFS_ARCH ?= "${DISTRO_ARCH}"
> ROOTFS_DISTRO ?= "${DISTRO}"
> ROOTFS_PACKAGES ?= ""
> @@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
> /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
> ${ROOTFS_PACKAGES}
> }
>
> -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
> -ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
> +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
> +ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
> +
> +ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
> +rootfs_export_package_cache[weight] = "5"
> +rootfs_export_package_cache() {
> + deb_dl_dir_export ${ROOTFSDIR}
> +}
>
> ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
> (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
> rootfs_install_clean_files[weight] = "2"
> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> index ab608da..d86c5b9 100644
> --- a/meta/conf/bitbake.conf
> +++ b/meta/conf/bitbake.conf
> @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
> FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
> FILESEXTRAPATHS ?= "__default:"
> GITDIR = "${DL_DIR}/git"
> +DEBDIR = "${DL_DIR}/deb"
> DEBSRCDIR = "${DL_DIR}/deb-src"
> P = "${PN}-${PV}"
> PF = "${PN}-${PV}-${PR}"
> --
> 2.24.1
>
>
[-- Attachment #1.2: Type: text/html, Size: 8738 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs
2020-01-25 14:06 ` vijai kumar
@ 2020-01-28 14:43 ` Henning Schild
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
0 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2020-01-28 14:43 UTC (permalink / raw)
To: vijai kumar; +Cc: isar-users
Hi,
thanks for the report. Since you already have a patch ready, can you
please share it? I would suggest to include it into the series, or
apply it on top. Or squash it in.
Henning
On Sat, 25 Jan 2020 06:06:59 -0800
vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
> On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild
> wrote:
> >
> > From: Henning Schild <hennin...@siemens.com <javascript:>>
> >
> > Hook in between the "apt-get --download-only"s and the "apt-get"s
> > and copy out all the debs for later.
> >
> > Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> > ---
> > meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
> > meta/classes/dpkg-base.bbclass | 1 +
> > meta/classes/dpkg-gbp.bbclass | 1 +
> > meta/classes/dpkg.bbclass | 1 +
> > meta/classes/image-locales-extension.bbclass | 2 +-
> > meta/classes/image-tools-extension.bbclass | 1 +
> > meta/classes/rootfs.bbclass | 12 ++++++++++--
> > meta/conf/bitbake.conf | 1 +
> > 8 files changed, 31 insertions(+), 3 deletions(-)
> > create mode 100644 meta/classes/deb-dl-dir.bbclass
> >
> > diff --git a/meta/classes/deb-dl-dir.bbclass
> > b/meta/classes/deb-dl-dir.bbclass
> > new file mode 100644
> > index 0000000..e41e981
> > --- /dev/null
> > +++ b/meta/classes/deb-dl-dir.bbclass
> > @@ -0,0 +1,15 @@
> > +# This software is a part of ISAR.
> > +# Copyright (C) 2019 Siemens AG
> > +#
> > +# SPDX-License-Identifier: MIT
> > +
> > +deb_dl_dir_export() {
> > + export pc="${DEBDIR}/${DISTRO}"
> > + export rootfs="${1}"
> > + mkdir -p "${pc}"
> > + flock "${pc}".lock -c ' \
> > + sudo find "${rootfs}"/var/cache/apt/archives/ -type f
> > -iname '*\.deb' \
> > + -exec cp -f '{}' "${pc}" \;
> > + sudo chown -R $(id -u):$(id -g) "${pc}"
> > + '
> > +}
> > diff --git a/meta/classes/dpkg-base.bbclass
> > b/meta/classes/dpkg-base.bbclass
> > index aa11660..fd0b07f 100644
> > --- a/meta/classes/dpkg-base.bbclass
> > +++ b/meta/classes/dpkg-base.bbclass
> > @@ -8,6 +8,7 @@ inherit buildchroot
> > inherit debianize
> > inherit terminal
> > inherit repository
> > +inherit deb-dl-dir
> >
> > DEPENDS ?= ""
> >
> > diff --git a/meta/classes/dpkg-gbp.bbclass
> > b/meta/classes/dpkg-gbp.bbclass index 2afe9db..7e908eb 100644
> > --- a/meta/classes/dpkg-gbp.bbclass
> > +++ b/meta/classes/dpkg-gbp.bbclass
> > @@ -15,6 +15,7 @@ do_install_builddeps_append() {
> > sudo -E chroot ${BUILDCHROOT_DIR} \
> > apt-get install -y -o Debug::pkgProblemResolver=yes \
> > --no-install-recommends --download-only
> > ${GBP_DEPENDS}
> > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > sudo -E chroot ${BUILDCHROOT_DIR} \
> > apt-get install -y -o Debug::pkgProblemResolver=yes \
> > --no-install-recommends ${GBP_DEPENDS}
> > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
> > index b26c645..7e2c86d 100644
> > --- a/meta/classes/dpkg.bbclass
> > +++ b/meta/classes/dpkg.bbclass
> > @@ -9,6 +9,7 @@ do_install_builddeps() {
>
>
>
> While testing this series in a downstream project, I do not see a
> part of KBUILD_DEPENDS cached.
>
> We don't have the "linux-custom recipe rework" changes by Cedric
> there. The old Linux recipe overrides
> this task with its own variant.
>
> I have added deb_dl_* there as well to fix the issue.
>
> This might become a problem when more such recipes pop-up. Unlikely,
> but possible.
> Recipe authors should know they have to take care of this if they
> decide to override
> do_install_builddeps.
>
>
> Thanks,
> Vijai Kumar K
>
>
> E="${@ isar_export_proxies(d)}"
> > sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
> > ${PP}/${PPS} ${DISTRO_ARCH} --download-only
> > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
> > ${PP}/${PPS} ${DISTRO_ARCH}
> > dpkg_undo_mounts
> > diff --git a/meta/classes/image-locales-extension.bbclass
> > b/meta/classes/image-locales-extension.bbclass
> > index 3c0758f..0f0d0ca 100644
> > --- a/meta/classes/image-locales-extension.bbclass
> > +++ b/meta/classes/image-locales-extension.bbclass
> > @@ -25,7 +25,7 @@ def get_nopurge(d):
> > j.split()[0].split(".")[0],
> > j.split()[0]))))
> >
> > -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN +=
> > "image_install_localepurge_download"
> > +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT +=
> > "image_install_localepurge_download"
> > image_install_localepurge_download[weight] = "40"
> > image_install_localepurge_download() {
> > sudo -E chroot '${ROOTFSDIR}' \
> > diff --git a/meta/classes/image-tools-extension.bbclass
> > b/meta/classes/image-tools-extension.bbclass
> > index b8531cb..ec67d94 100644
> > --- a/meta/classes/image-tools-extension.bbclass
> > +++ b/meta/classes/image-tools-extension.bbclass
> > @@ -34,6 +34,7 @@ do_install_imager_deps() {
> > --allow-unauthenticated --allow-downgrades
> > --download-only install \
> > ${IMAGER_INSTALL}'
> >
> > + deb_dl_dir_export ${BUILDCHROOT_DIR}
> > sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
> > apt-get -o Debug::pkgProblemResolver=yes
> > --no-install-recommends -y \
> > --allow-unauthenticated --allow-downgrades install \
> > diff --git a/meta/classes/rootfs.bbclass
> > b/meta/classes/rootfs.bbclass index b4dbfa3..a6b6110 100644
> > --- a/meta/classes/rootfs.bbclass
> > +++ b/meta/classes/rootfs.bbclass
> > @@ -1,6 +1,8 @@
> > # This software is a part of ISAR.
> > # Copyright (c) Siemens AG, 2019
> >
> > +inherit deb-dl-dir
> > +
> > ROOTFS_ARCH ?= "${DISTRO_ARCH}"
> > ROOTFS_DISTRO ?= "${DISTRO}"
> > ROOTFS_PACKAGES ?= ""
> > @@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
> > /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only
> > ${ROOTFS_PACKAGES}
> > }
> >
> > -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
> > -ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
> > +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
> > +ROOTFS_INSTALL_COMMAND +=
> > "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
> > +
> > +ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
> > +rootfs_export_package_cache[weight] = "5"
> > +rootfs_export_package_cache() {
> > + deb_dl_dir_export ${ROOTFSDIR}
> > +}
> >
> > ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if
> > (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
> > rootfs_install_clean_files[weight] = "2"
> > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > index ab608da..d86c5b9 100644
> > --- a/meta/conf/bitbake.conf
> > +++ b/meta/conf/bitbake.conf
> > @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
> > FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
> > FILESEXTRAPATHS ?= "__default:"
> > GITDIR = "${DL_DIR}/git"
> > +DEBDIR = "${DL_DIR}/deb"
> > DEBSRCDIR = "${DL_DIR}/deb-src"
> > P = "${PN}-${PV}"
> > PF = "${PN}-${PV}-${PR}"
> > --
> > 2.24.1
> >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Base-apt rework v3
2020-01-28 14:43 ` Henning Schild
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 01/26] repository: new class to deal with repos Vijai Kumar K
` (25 more replies)
0 siblings, 26 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users
This V3 includes all of Henning's V2 patches + a fix for caching
KBUILD_DEPENDS.
Henning,
The last patch in the series(the fix) would not be needed once you
rebase to next. The do_install_builddeps is dropped in the new
linux recipe rework. Anyway sending it across.
Also, These patches are tested for its use case.
Tested-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
Thanks,
Vijai Kumar K
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 01/26] repository: new class to deal with repos
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 02/26] dpkg-base: add download caching of apt:// downloads Vijai Kumar K
` (24 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Factor out all the "reprepro" code into a common class. There are slight
functional changes since the two copies of the code got out of sync.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 28 ++++----
meta/classes/dpkg-base.bbclass | 24 ++-----
meta/classes/repository.bbclass | 69 +++++++++++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 37 +++-------
.../base-apt/files/distributions.in | 3 -
.../isar-apt/files/distributions.in | 3 -
meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++-----
7 files changed, 102 insertions(+), 88 deletions(-)
create mode 100644 meta/classes/repository.bbclass
delete mode 100644 meta/recipes-devtools/base-apt/files/distributions.in
delete mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 90b2cfc..1f08c70 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -1,5 +1,10 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+inherit repository
compare_pkg_md5sums() {
pkg1=$1
@@ -14,10 +19,6 @@ compare_pkg_md5sums() {
populate_base_apt() {
search_dir=$1
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
find $search_dir -name '*.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
@@ -40,18 +41,15 @@ populate_base_apt() {
compare_pkg_md5sums "$package" "$base_apt_p" && continue
# md5sum differs, so remove the package from base-apt
- name=$(echo $base_name | cut -d '_' -f 1)
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main -A ${DISTRO_ARCH} \
- remove ${BASE_DISTRO_CODENAME} \
- $name
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
fi
- reprepro -b ${REPO_BASE_DIR}/${BASE_DISTRO} \
- --dbdir ${REPO_BASE_DB_DIR}/${BASE_DISTRO} \
- -C main \
- includedeb ${BASE_DISTRO_CODENAME} \
- $package
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
done
}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index af9c984..f38168c 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -7,6 +7,7 @@
inherit buildchroot
inherit debianize
inherit terminal
+inherit repository
DEPENDS ?= ""
@@ -140,31 +141,16 @@ repo_clean() {
DEBS=$( find ${S}/.. -maxdepth 1 -name "*.deb" || [ ! -d ${S} ] )
if [ -n "${DEBS}" ]; then
for d in ${DEBS}; do
- p=$( dpkg-deb --show --showformat '${Package}' ${d} )
- a=$( dpkg-deb --show --showformat '${Architecture}' ${d} )
- # removing "all" means no arch
- aarg="-A ${a}"
- [ "${a}" = "all" ] && aarg=""
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main ${aarg} \
- remove ${DEBDISTRONAME} \
- ${p}
+ repo_del_package "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" "${d}"
done
fi
}
-# Install package to Isar-apt
do_deploy_deb() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
repo_clean
- reprepro -b ${REPO_ISAR_DIR}/${DISTRO} \
- --dbdir ${REPO_ISAR_DB_DIR}/${DISTRO} \
- -C main \
- includedeb ${DEBDISTRONAME} \
- ${S}/../*.deb
+ repo_add_packages "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" "${DEBDISTRONAME}" ${S}/../*.deb
}
addtask deploy_deb after do_dpkg_build before do_build
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
new file mode 100644
index 0000000..880ddef
--- /dev/null
+++ b/meta/classes/repository.bbclass
@@ -0,0 +1,69 @@
+# This software is a part of ISAR.
+# Copyright (C) 2017-2019 Siemens AG
+# Copyright (C) 2019 ilbers GmbH
+#
+# SPDX-License-Identifier: MIT
+
+repo_create() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local keyfiles="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+
+ if [ ! -f "${dir}"/conf/distributions ]; then
+ mkdir -p "${dir}"/conf/
+ cat <<EOF > "${dir}"/conf/distributions
+Codename: ${codename}
+Architectures: i386 armhf arm64 amd64 mipsel source
+Components: main
+EOF
+ if [ -n "${keyfiles}" ] ; then
+ local option=""
+ for key in ${keyfiles}; do
+ keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
+ option="${option}${keyid} "
+ done
+ echo "SignWith: ${option}" >> "${dir}"/conf/distributions
+ fi
+ fi
+ if [ ! -d "${dbdir}" ]; then
+ reprepro -b "${dir}" --dbdir "${dbdir}" export "${codename}"
+ fi
+}
+
+repo_add_packages() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main \
+ includedeb "${codename}" \
+ "$@"
+}
+
+repo_del_package() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ local file="$4"
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ local p=$( dpkg-deb --show --showformat '${Package}' "${file}" )
+ local a=$( dpkg-deb --show --showformat '${Architecture}' "${file}" )
+ # removing "all" means no arch
+ local aarg="-A ${a}"
+ [ "${a}" = "all" ] && aarg=""
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main ${aarg} \
+ remove "${codename}" \
+ "${p}"
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 9a0f7c8..c9a7dad 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -1,45 +1,24 @@
# This software is a part of ISAR.
# Copyright (C) 2018 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-CACHE_CONF_DIR = "${REPO_BASE_DIR}/${BASE_DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
repo_config() {
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
-
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- if [ -n "${KEYFILES}" ]; then
- option=""
- for key in ${KEYFILES}; do
- keyid=$(gpg --keyid-format 0xlong --with-colons ${key} 2>/dev/null | grep "^pub:" | awk -F':' '{print $5;}')
- option="${option}${keyid} "
- done
- # To generate Release.gpg
- echo "SignWith: ${option}" >> ${CACHE_CONF_DIR}/distributions
- fi
- fi
-
- path_cache="${REPO_BASE_DIR}/${BASE_DISTRO}"
- path_databases="${REPO_BASE_DB_DIR}/${BASE_DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${BASE_DISTRO_CODENAME}
- fi
+ repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${KEYFILES}"
}
python do_cache_config() {
diff --git a/meta/recipes-devtools/base-apt/files/distributions.in b/meta/recipes-devtools/base-apt/files/distributions.in
deleted file mode 100644
index fd7de0b..0000000
--- a/meta/recipes-devtools/base-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {CODENAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/files/distributions.in b/meta/recipes-devtools/isar-apt/files/distributions.in
deleted file mode 100644
index a24a700..0000000
--- a/meta/recipes-devtools/isar-apt/files/distributions.in
+++ /dev/null
@@ -1,3 +0,0 @@
-Codename: {DISTRO_NAME}
-Architectures: i386 armhf arm64 amd64 mipsel source
-Components: main
diff --git a/meta/recipes-devtools/isar-apt/isar-apt.bb b/meta/recipes-devtools/isar-apt/isar-apt.bb
index e6f1753..0227c1b 100644
--- a/meta/recipes-devtools/isar-apt/isar-apt.bb
+++ b/meta/recipes-devtools/isar-apt/isar-apt.bb
@@ -1,32 +1,20 @@
# This software is a part of ISAR.
# Copyright (C) 2015-2017 ilbers GmbH
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
-SRC_URI = "file://distributions.in"
+inherit repository
-CACHE_CONF_DIR = "${REPO_ISAR_DIR}/${DISTRO}/conf"
-do_cache_config[dirs] = "${CACHE_CONF_DIR}"
do_cache_config[stamp-extra-info] = "${DISTRO}"
do_cache_config[lockfiles] = "${REPO_ISAR_DIR}/isar.lock"
# Generate reprepro config for current distro if it doesn't exist. Once it's
# generated, this task should do nothing.
do_cache_config() {
- if [ ! -e "${CACHE_CONF_DIR}/distributions" ]; then
- sed -e "s#{DISTRO_NAME}#"${DEBDISTRONAME}"#g" \
- ${WORKDIR}/distributions.in > ${CACHE_CONF_DIR}/distributions
- fi
-
- path_cache="${REPO_ISAR_DIR}/${DISTRO}"
- path_databases="${REPO_ISAR_DB_DIR}/${DISTRO}"
-
- if [ ! -d "${path_databases}" ]; then
- if [ -n "${GNUPGHOME}" ]; then
- export GNUPGHOME="${GNUPGHOME}"
- fi
- reprepro -b ${path_cache} \
- --dbdir ${path_databases} \
- export ${DEBDISTRONAME}
- fi
+ repo_create "${REPO_ISAR_DIR}"/"${DISTRO}" \
+ "${REPO_ISAR_DB_DIR}"/"${DISTRO}" \
+ "${DEBDISTRONAME}"
}
addtask cache_config after do_unpack before do_build
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 02/26] dpkg-base: add download caching of apt:// downloads
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 01/26] repository: new class to deal with repos Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 03/26] meta: dpkg-base: convert "exit" into "return 0" Vijai Kumar K
` (23 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Store the results of "apt-get source" in the DL_DIR. This means we
really only need to fetch once on incremental rebuilds or if multiple
recipes use the same SRC_URI.
We can also later collect the results and put them in a repository.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
meta/conf/bitbake.conf | 1 +
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index f38168c..7a97d91 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -63,14 +63,30 @@ do_apt_fetch() {
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
- sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
- sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}'
+ for uri in "${SRC_APT}"; do
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
+ sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}"
+ done
+
dpkg_undo_mounts
}
addtask apt_fetch after do_unpack before do_patch
do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
+addtask cleanall_apt before do_cleanall
+do_cleanall_apt[nostamp] = "1"
+do_cleanall_apt() {
+ if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
+ exit
+ fi
+ for uri in "${SRC_APT}"; do
+ rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
+ done
+}
+
def get_package_srcdir(d):
s = os.path.abspath(d.getVar("S", True))
workdir = os.path.abspath(d.getVar("WORKDIR", True))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 0eb3b54..ab608da 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 03/26] meta: dpkg-base: convert "exit" into "return 0"
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 01/26] repository: new class to deal with repos Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 02/26] dpkg-base: add download caching of apt:// downloads Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 04/26] base-apt: change the sources.list to also offer deb-src Vijai Kumar K
` (22 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Bitbake seems to execute all the functions in subshells, but in fact we
mean to return here so correct the style.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-base.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index 7a97d91..aa11660 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -53,7 +53,7 @@ SRC_APT ?= ""
do_apt_fetch() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
rm -rf ${S}
dpkg_do_mounts
@@ -80,7 +80,7 @@ addtask cleanall_apt before do_cleanall
do_cleanall_apt[nostamp] = "1"
do_cleanall_apt() {
if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
- exit
+ return 0
fi
for uri in "${SRC_APT}"; do
rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 04/26] base-apt: change the sources.list to also offer deb-src
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (2 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 03/26] meta: dpkg-base: convert "exit" into "return 0" Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 05/26] base-apt: add deb-src packages as well Vijai Kumar K
` (21 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The repository is ready for it already. For the moment there will be
nothing there.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-core/isar-bootstrap/files/base-apt-sources | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
index ddc4509..2a4ab2c 100644
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
@@ -1 +1,2 @@
deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
+deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 05/26] base-apt: add deb-src packages as well
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (3 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 04/26] base-apt: change the sources.list to also offer deb-src Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 06/26] base-apt: do not skip gpg check when it is signed Vijai Kumar K
` (20 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
So far the deb-src packages downloaded via "apt://" where not cached.
This commit fixes that and includes those packages into the cache.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 7 +++++++
meta/classes/image-cache-extension.bbclass | 4 ++++
meta/classes/repository.bbclass | 14 ++++++++++++++
3 files changed, 25 insertions(+)
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index 1f08c70..db25adc 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -52,4 +52,11 @@ populate_base_apt() {
"${BASE_DISTRO_CODENAME}" \
"${package}"
done
+
+ find $search_dir -name '*.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 30db549..f8b9ba3 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -32,5 +32,9 @@ do_cache_base_repo() {
if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+
+ if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
+ fi
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 880ddef..dc4c1dc 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -35,6 +35,20 @@ EOF
fi
}
+repo_add_srcpackage() {
+ local dir="$1"
+ local dbdir="$2"
+ local codename="$3"
+ shift; shift; shift
+
+ if [ -n "${GNUPGHOME}" ]; then
+ export GNUPGHOME="${GNUPGHOME}"
+ fi
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ includedsc "${codename}" \
+ "$@"
+}
+
repo_add_packages() {
local dir="$1"
local dbdir="$2"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 06/26] base-apt: do not skip gpg check when it is signed
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (4 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 05/26] base-apt: add deb-src packages as well Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 07/26] ci: conf: add "hello" to the sample config and every build Vijai Kumar K
` (19 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We used to have "trusted=yes" hardcoded even for a signed base-apt. Make
that flag depend on whether it is signed and generate that file with two
echos, it is simple enough.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
.../isar-bootstrap/files/base-apt-sources | 2 --
.../isar-bootstrap/isar-bootstrap.inc | 16 ++++++++--------
2 files changed, 8 insertions(+), 10 deletions(-)
delete mode 100644 meta/recipes-core/isar-bootstrap/files/base-apt-sources
diff --git a/meta/recipes-core/isar-bootstrap/files/base-apt-sources b/meta/recipes-core/isar-bootstrap/files/base-apt-sources
deleted file mode 100644
index 2a4ab2c..0000000
--- a/meta/recipes-core/isar-bootstrap/files/base-apt-sources
+++ /dev/null
@@ -1,2 +0,0 @@
-deb [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
-deb-src [trusted=yes] file:///base-apt/{BASE_DISTRO} {BASE_DISTRO_CODENAME} main
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5dd571e..cfcf7b8 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -12,8 +12,7 @@ SRC_URI = " \
file://isar-apt.conf \
file://isar-apt-fallback.conf \
file://locale \
- file://chroot-setup.sh \
- file://base-apt-sources"
+ file://chroot-setup.sh"
PV = "1.0"
DEBOOTSTRAP ?= "qemu-debootstrap"
@@ -21,7 +20,6 @@ ROOTFSDIR = "${WORKDIR}/rootfs"
APTPREFS = "${WORKDIR}/apt-preferences"
APTSRCS = "${WORKDIR}/apt-sources"
APTSRCS_INIT = "${WORKDIR}/apt-sources-init"
-BASEAPTSRCS = "${WORKDIR}/base-apt-sources"
DISTRO_BOOTSTRAP_KEYFILES = ""
THIRD_PARTY_APT_KEYFILES = ""
DEPLOY_ISAR_BOOTSTRAP ?= ""
@@ -266,13 +264,15 @@ isar_bootstrap() {
"${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
- sed -e "s#{BASE_DISTRO}#"${BASE_DISTRO}"#g" \
- -e "s#{BASE_DISTRO_CODENAME}#"${BASE_DISTRO_CODENAME}"#g" \
- -i ${BASEAPTSRCS}
+ line="file:///base-apt/${BASE_DISTRO} ${BASE_DISTRO_CODENAME} main"
+ if [ -z "${BASE_REPO_KEY}" ]; then
+ line="[trusted=yes] ${line}"
+ fi
+ echo "deb ${line}" > "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+ echo "deb-src ${line}" >> "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
mkdir -p ${ROOTFSDIR}/base-apt
sudo mount --bind ${REPO_BASE_DIR} ${ROOTFSDIR}/base-apt
- install -v -m644 "${BASEAPTSRCS}" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
else
install -v -m644 "${APTSRCS}" \
"${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 07/26] ci: conf: add "hello" to the sample config and every build
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (5 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 06/26] base-apt: do not skip gpg check when it is signed Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 08/26] meta: split all apt-get invocations into download and execution Vijai Kumar K
` (18 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Now that base-apt supports deb-src we can include hello in a standard
build.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 280e007..0e55da1 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,11 +166,9 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
## "cache_base_repo" relies on the cache ...
#IMAGE_INSTALL += "isar-disable-apt-cache"
-## uses apt:// SRC_URI, which does not work with the cache
-#IMAGE_INSTALL += "hello"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7cd5a2f..f50c9e4 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -169,7 +169,6 @@ if [ -n "$REPRO_BUILD" ]; then
fi
sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-sed -i -e 's/#IMAGE_INSTALL += "hello"/IMAGE_INSTALL += "hello"/g' conf/local.conf
echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 08/26] meta: split all apt-get invocations into download and execution
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (6 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 07/26] ci: conf: add "hello" to the sample config and every build Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 09/26] meta: create DL_DIR support for all apt-get downloaded .debs Vijai Kumar K
` (17 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
This commit does not contain a functional change. We just split all
calls of "apt-get" into a download and an execution phase, so we can
later copy out the downloaded files.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/dpkg-gbp.bbclass | 3 ++
meta/classes/dpkg.bbclass | 5 ++-
meta/classes/image-tools-extension.bbclass | 5 +++
.../buildchroot/files/deps.sh | 37 +++++++++++--------
4 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 0533a52..2afe9db 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,9 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
do_install_builddeps_append() {
dpkg_do_mounts
+ sudo -E chroot ${BUILDCHROOT_DIR} \
+ apt-get install -y -o Debug::pkgProblemResolver=yes \
+ --no-install-recommends --download-only ${GBP_DEPENDS}
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 180a3a8..b26c645 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,7 +7,10 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
- sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh ${PP}/${PPS} ${DISTRO_ARCH}
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
+ ${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
}
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index d4f216d..b8531cb 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -30,6 +30,11 @@ do_install_imager_deps() {
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
-o Dir::Etc::SourceParts="-" \
-o APT::Get::List-Cleanup="0"
+ apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
+ --allow-unauthenticated --allow-downgrades --download-only install \
+ ${IMAGER_INSTALL}'
+
+ sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
${IMAGER_INSTALL}'
diff --git a/meta/recipes-devtools/buildchroot/files/deps.sh b/meta/recipes-devtools/buildchroot/files/deps.sh
index 002085b..0f11d78 100644
--- a/meta/recipes-devtools/buildchroot/files/deps.sh
+++ b/meta/recipes-devtools/buildchroot/files/deps.sh
@@ -12,27 +12,34 @@ source /isar/common.sh
# 2) we add -y to go non-interactive
# 3) downgrades shall be allowed in case a package recipe was changed
install_cmd="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends \
- -y --allow-downgrades"
+ -y --allow-downgrades $3"
-# Make sure that we have latest isar-apt content.
-# Options meaning:
-# Dir::Etc::SourceList - specifies which source to be used
-# Dir::Etc::SourceParts - disables looking for the other sources
-# APT::Get::List-Cleanup - do not erase obsolete packages list for
-# upstream in '/var/lib/apt/lists'
-apt-get update \
- -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
- -o Dir::Etc::SourceParts="-" \
- -o APT::Get::List-Cleanup="0"
+if [ "$3" != "-download-only" ]; then
+ # Make sure that we have latest isar-apt content.
+ # Options meaning:
+ # Dir::Etc::SourceList - specifies which source to be used
+ # Dir::Etc::SourceParts - disables looking for the other sources
+ # APT::Get::List-Cleanup - do not erase obsolete packages list for
+ # upstream in '/var/lib/apt/lists'
+ apt-get update \
+ -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
+ -o Dir::Etc::SourceParts="-" \
+ -o APT::Get::List-Cleanup="0"
+fi
# Do not set an architecture when building only 'all' (generic) packages.
# This can avoid unneeded cross-build issues.
if ! grep "^Architecture:" debian/control | grep -qv "all"; then
- set_arch=""
+ set_arch=""
fi
# Install all build deps
-mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
+if [ "$3" = "--download-only" ]; then
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control 2>&1 \
+ | grep "mk-build-deps: Unable to install all build-dep packages"
+else
+ mk-build-deps $set_arch -t "${install_cmd}" -i -r debian/control
-# Upgrade any already installed packages in case we are partially rebuilding
-apt-get upgrade -y --allow-downgrades
+ # Upgrade any already installed packages in case we are partially rebuilding
+ apt-get upgrade -y --allow-downgrades
+fi
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 09/26] meta: create DL_DIR support for all apt-get downloaded .debs
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (7 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 08/26] meta: split all apt-get invocations into download and execution Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 10/26] meta: import DL_DIR debs before apt-get download steps Vijai Kumar K
` (16 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Hook in between the "apt-get --download-only"s and the "apt-get"s and
copy out all the debs for later.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++
meta/classes/dpkg-base.bbclass | 1 +
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-locales-extension.bbclass | 2 +-
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 12 ++++++++++--
meta/conf/bitbake.conf | 1 +
8 files changed, 31 insertions(+), 3 deletions(-)
create mode 100644 meta/classes/deb-dl-dir.bbclass
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
new file mode 100644
index 0000000..e41e981
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@
+# This software is a part of ISAR.
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+deb_dl_dir_export() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ mkdir -p "${pc}"
+ flock "${pc}".lock -c ' \
+ sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
+ -exec cp -f '{}' "${pc}" \;
+ sudo chown -R $(id -u):$(id -g) "${pc}"
+ '
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index aa11660..fd0b07f 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
inherit debianize
inherit terminal
inherit repository
+inherit deb-dl-dir
DEPENDS ?= ""
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 2afe9db..7e908eb 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index b26c645..7e2c86d 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
E="${@ isar_export_proxies(d)}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH} --download-only
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH}
dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/classes/image-locales-extension.bbclass
index 3c0758f..0f0d0ca 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
j.split()[0].split(".")[0],
j.split()[0]))))
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += "image_install_localepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += "image_install_localepurge_download"
image_install_localepurge_download[weight] = "40"
image_install_localepurge_download() {
sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index b8531cb..ec67d94 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'
+ deb_dl_dir_export ${BUILDCHROOT_DIR}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index b4dbfa3..a6b6110 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,6 +1,8 @@
# This software is a part of ISAR.
# Copyright (c) Siemens AG, 2019
+inherit deb-dl-dir
+
ROOTFS_ARCH ?= "${DISTRO_ARCH}"
ROOTFS_DISTRO ?= "${DISTRO}"
ROOTFS_PACKAGES ?= ""
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
/usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only ${ROOTFS_PACKAGES}
}
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= ""
-ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= ""
+ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
+
+ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] = "5"
+rootfs_export_package_cache() {
+ deb_dl_dir_export ${ROOTFSDIR}
+}
ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
rootfs_install_clean_files[weight] = "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index ab608da..d86c5b9 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy"
FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}"
FILESEXTRAPATHS ?= "__default:"
GITDIR = "${DL_DIR}/git"
+DEBDIR = "${DL_DIR}/deb"
DEBSRCDIR = "${DL_DIR}/deb-src"
P = "${PN}-${PV}"
PF = "${PN}-${PV}-${PR}"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 10/26] meta: import DL_DIR debs before apt-get download steps
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (8 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 09/26] meta: create DL_DIR support for all apt-get downloaded .debs Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 11/26] meta: include DL_DIR deb import/export into bootstrap Vijai Kumar K
` (15 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
If we ever downloaded a file there is no need to do that again. We can
not only use that DL_DIR to later construct a repository but also to
speed up our rebuilds.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 10 ++++++++++
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 6 ++++++
5 files changed, 19 insertions(+)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index e41e981..ab4b1f0 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,6 +3,16 @@
#
# SPDX-License-Identifier: MIT
+deb_dl_dir_import() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ [ ! -d "${pc}" ] && return 0
+ flock -s "${pc}".lock -c ' \
+ sudo find "${pc}" -type f -iname '*\.deb' -exec \
+ cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ '
+}
+
deb_dl_dir_export() {
export pc="${DEBDIR}/${DISTRO}"
export rootfs="${1}"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 7e908eb..afa1e19 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
do_install_builddeps_append() {
dpkg_do_mounts
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7e2c86d..585365e 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,6 +7,7 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH} --download-only
deb_dl_dir_export "${BUILDCHROOT_DIR}"
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index ec67d94..6590ee7 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,6 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts
E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import ${BUILDCHROOT_DIR}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get update \
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index a6b6110..eac553c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
fi
}
+ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
+rootfs_import_package_cache[weight] = "5"
+rootfs_import_package_cache() {
+ deb_dl_dir_import ${ROOTFSDIR}
+}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 11/26] meta: include DL_DIR deb import/export into bootstrap
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (9 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 10/26] meta: import DL_DIR debs before apt-get download steps Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:41 ` [PATCH v3 12/26] base-apt: populate from DEBDIR as well Vijai Kumar K
` (14 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The bootstrap step also contains downloading of debs, so it should be
surrounded with an import/export as well.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cfcf7b8..5b14cd3 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -28,6 +28,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = ",apt-transport-https,ca-certificates"
HOST_DISTRO_APT_SOURCES += "conf/distro/${HOST_DISTRO}.list"
+inherit deb-dl-dir
+
python () {
distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
@@ -219,6 +221,7 @@ do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
isar_bootstrap() {
+ deb_dl_dir_import "${ROOTFSDIR}"
IS_HOST=""
while true; do
case "$1" in
@@ -335,6 +338,7 @@ isar_bootstrap() {
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
fi
EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}"
}
CLEANFUNCS = "clean_deploy"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 12/26] base-apt: populate from DEBDIR as well
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (10 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 11/26] meta: include DL_DIR deb import/export into bootstrap Vijai Kumar K
@ 2020-01-28 15:41 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 13/26] base-apt: drop the "apt_cache" feature Vijai Kumar K
` (13 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:41 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We have all the .debs cached in DEBDIR now, take them from there because
the rootfss might not actually have all of them.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index f8b9ba3..07e9e7d 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -33,6 +33,10 @@ do_cache_base_repo() {
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+ if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBDIR}"/"${DISTRO}"
+ fi
+
if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
fi
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 13/26] base-apt: drop the "apt_cache" feature
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (11 preceding siblings ...)
2020-01-28 15:41 ` [PATCH v3 12/26] base-apt: populate from DEBDIR as well Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 14/26] base-apt: do not copy debs directly out of rootfss anymore Vijai Kumar K
` (12 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We now have all debs in DEBDIR anyways, no need to conditionally copy
them out of the rootfss we create.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 4 ----
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 10 ----------
3 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 07e9e7d..19c9c59 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,10 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d ${WORKDIR}/apt_cache ]; then
- populate_base_apt ${WORKDIR}/apt_cache
- fi
-
if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
fi
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 725bc04..b687fcf 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,7 +60,7 @@ image_do_mounts() {
}
ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index eac553c..6d4d42c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -9,7 +9,6 @@ ROOTFS_PACKAGES ?= ""
# Features of the rootfs creation:
# available features are:
-# 'copy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
@@ -186,15 +185,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack
-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
-rootfs_postprocess_copy_package_cache() {
- mkdir -p '${WORKDIR}/apt_cache'
- sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
- me="$(id -u):$(id -g)"
- sudo chown -R "$me" '${WORKDIR}/apt_cache'
-}
-
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {
sudo -E chroot '${ROOTFSDIR}' \
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 14/26] base-apt: do not copy debs directly out of rootfss anymore
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (12 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 13/26] base-apt: drop the "apt_cache" feature Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 15/26] base-apt: rework base-apt population Vijai Kumar K
` (11 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The earlier commits make sure we already have all .debs in the repo, no
need to get them from these locations. In fact these locations are not
reliable anyways, see hook in "isar-disable-apt-cache".
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 8 --------
1 file changed, 8 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 19c9c59..cff2a8b 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,14 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
- fi
-
- if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
- fi
-
if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
populate_base_apt "${DEBDIR}"/"${DISTRO}"
fi
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 15/26] base-apt: rework base-apt population
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (13 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 14/26] base-apt: do not copy debs directly out of rootfss anymore Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 16/26] base-apt: move class "base-apt-helper" into only user Vijai Kumar K
` (10 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Just call that function once and make it find the debs and dscs in one
run.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 6 ++----
meta/classes/image-cache-extension.bbclass | 8 +-------
2 files changed, 3 insertions(+), 11 deletions(-)
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index db25adc..18d43aa 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -17,9 +17,7 @@ compare_pkg_md5sums() {
}
populate_base_apt() {
- search_dir=$1
-
- find $search_dir -name '*.deb' | while read package; do
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
# addition, md5sums are compared to ensure that the package is the
@@ -53,7 +51,7 @@ populate_base_apt() {
"${package}"
done
- find $search_dir -name '*.dsc' | while read package; do
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index cff2a8b..7945e8b 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,12 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBDIR}"/"${DISTRO}"
- fi
-
- if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
- fi
+ populate_base_apt
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 16/26] base-apt: move class "base-apt-helper" into only user
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (14 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 15/26] base-apt: rework base-apt population Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 17/26] CI: include "isar-disable-apt-cache" into all CI images Vijai Kumar K
` (9 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
There is only one place we use that code, move it there instead of
having an extra class.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 60 ----------------------
meta/classes/image-cache-extension.bbclass | 55 +++++++++++++++++++-
2 files changed, 54 insertions(+), 61 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
deleted file mode 100644
index 18d43aa..0000000
--- a/meta/classes/base-apt-helper.bbclass
+++ /dev/null
@@ -1,60 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) 2018 ilbers GmbH
-# Copyright (C) 2019 Siemens AG
-#
-# SPDX-License-Identifier: MIT
-
-inherit repository
-
-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
-
- # Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 7945e8b..c3ee7b8 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -5,7 +5,60 @@
#
# This class extends the image.bbclass to supply the creation of cache repositories
-inherit base-apt-helper
+inherit repository
+
+compare_pkg_md5sums() {
+ pkg1=$1
+ pkg2=$2
+
+ md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
+ md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
+
+ [ "$md1" = "$md2" ]
+}
+
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
+
+ # Check if this package is taken from Isar-apt, if so - ignore it.
+ base_name=${package##*/}
+ isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
+ if [ -n "$isar_apt_p" ]; then
+ # Check if MD5 sums are identical. This helps to avoid the case
+ # when packages is overridden from another repo.
+ compare_pkg_md5sums "$package" "$isar_apt_p" && continue
+ fi
+
+ # Check if this package is already in base-apt
+ base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
+ if [ -n "$base_apt_p" ]; then
+ compare_pkg_md5sums "$package" "$base_apt_p" && continue
+
+ # md5sum differs, so remove the package from base-apt
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
do_cache_base_repo[depends] = "base-apt:do_cache_config"
do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 17/26] CI: include "isar-disable-apt-cache" into all CI images
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (15 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 16/26] base-apt: move class "base-apt-helper" into only user Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 18/26] CI: include "cowsay" into default build to test dpkg-gbp Vijai Kumar K
` (8 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The base-apt does not rely on the rootfs-cache anymore, so we can test
our cleanup package in all images.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 0e55da1..dee1250 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,9 +166,7 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
-## "cache_base_repo" relies on the cache ...
-#IMAGE_INSTALL += "isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index f50c9e4..bfaf1f6 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi
-sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-
echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
# Start cross build for the defined set of configurations
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 18/26] CI: include "cowsay" into default build to test dpkg-gbp
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (16 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 17/26] CI: include "isar-disable-apt-cache" into all CI images Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 19/26] CI: set BB_NO_NETWORK for cached rebuild Vijai Kumar K
` (7 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 2 +-
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index dee1250..9237bb3 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,7 +166,7 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache cowsay"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index bfaf1f6..d440c24 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi
-echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
-
# Start cross build for the defined set of configurations
sed -i -e 's/ISAR_CROSS_COMPILE ?= "0"/ISAR_CROSS_COMPILE ?= "1"/g' conf/local.conf
bitbake $BB_ARGS $CROSS_TARGETS_SET
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 19/26] CI: set BB_NO_NETWORK for cached rebuild
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (17 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 18/26] CI: include "cowsay" into default build to test dpkg-gbp Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 20/26] meta: repository: implement repo_contains_package and use it in base_apt Vijai Kumar K
` (6 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Also add it to our sample config and comment on its use in conjuction
with base-apt.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
doc/user_manual.md | 1 +
meta-isar/conf/local.conf.sample | 4 ++++
scripts/ci_build.sh | 4 ++++
3 files changed, 9 insertions(+)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index c9062d6..4d699bd 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -833,6 +833,7 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
```
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#BB_NO_NETWORK ?= "1"
```
- Remove build artifacts to use only local base-apt:
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 9237bb3..8355a40 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -176,6 +176,10 @@ ISAR_CROSS_COMPILE ?= "0"
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#
+# You probably want to uncomment this as well to make sure the build
+# does not access the network
+#BB_NO_NETWORK ?= "1"
# Set root password to 'root'
# Password was encrypted using following command:
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index d440c24..7da3ee9 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -149,11 +149,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of signed cached base repository
sudo rm -rf tmp
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
# Enable use of unsigned cached base repository
@@ -161,11 +163,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of unsigned cached base repository
sudo rm -rf tmp
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
fi
# Start cross build for the defined set of configurations
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 20/26] meta: repository: implement repo_contains_package and use it in base_apt
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (18 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 19/26] CI: set BB_NO_NETWORK for cached rebuild Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 21/26] meta: repository: simplify the matching of packages Vijai Kumar K
` (5 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Make the lookup and the checksum comparison one function of a
repository. That cleans things up and allows for easier reuse.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 31 ++++++----------------
meta/classes/repository.bbclass | 19 +++++++++++++
2 files changed, 27 insertions(+), 23 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index c3ee7b8..4123326 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -7,16 +7,6 @@
inherit repository
-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
populate_base_apt() {
find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
@@ -25,21 +15,16 @@ populate_base_apt() {
# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.
- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
# Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index dc4c1dc..e7acf59 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -81,3 +81,22 @@ repo_del_package() {
remove "${codename}" \
"${p}"
}
+
+repo_contains_package() {
+ local dir="$1"
+ local file="$2"
+ local package
+
+ package=$(find ${dir} -name ${file##*/})
+ if [ -n "$package" ]; then
+ local md1=$(md5sum "$package" | cut -d ' ' -f 1)
+ local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
+
+ # yes
+ [ "${md1}" = "${md2}" ] && return 0
+ # yes but not the exact same file
+ return 1
+ fi
+ # no
+ return 2
+}
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 21/26] meta: repository: simplify the matching of packages
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (19 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 20/26] meta: repository: implement repo_contains_package and use it in base_apt Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 22/26] meta: deb-dl-dir: do not cache debs from isar-apt Vijai Kumar K
` (4 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Just call cmp once instead of comparing the md5 sums.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/repository.bbclass | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index e7acf59..de1fa5e 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -89,11 +89,9 @@ repo_contains_package() {
package=$(find ${dir} -name ${file##*/})
if [ -n "$package" ]; then
- local md1=$(md5sum "$package" | cut -d ' ' -f 1)
- local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
-
# yes
- [ "${md1}" = "${md2}" ] && return 0
+ cmp --silent "$package" "$file" && return 0
+
# yes but not the exact same file
return 1
fi
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 22/26] meta: deb-dl-dir: do not cache debs from isar-apt
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (20 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 21/26] meta: repository: simplify the matching of packages Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 23/26] base-apt: pull base-apt population to the front of the build chain Vijai Kumar K
` (3 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Packages from isar-apt are not downloaded from the outside and should
not be cached.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index ab4b1f0..f969960 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,8 +3,10 @@
#
# SPDX-License-Identifier: MIT
+inherit repository
+
deb_dl_dir_import() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
[ ! -d "${pc}" ] && return 0
flock -s "${pc}".lock -c ' \
@@ -14,12 +16,16 @@ deb_dl_dir_import() {
}
deb_dl_dir_export() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
mkdir -p "${pc}"
flock "${pc}".lock -c ' \
- sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
- -exec cp -f '{}' "${pc}" \;
+ find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ while read p; do
+ repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
+ continue
+ sudo cp -f "${p}" "${pc}"
+ done
sudo chown -R $(id -u):$(id -g) "${pc}"
'
}
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 23/26] base-apt: pull base-apt population to the front of the build chain
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (21 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 22/26] meta: deb-dl-dir: do not cache debs from isar-apt Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 24/26] meta: base-apt: remove isar-apt check from population Vijai Kumar K
` (2 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.
The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass
diff --git a/doc/user_manual.md b/doc/user_manual.md
index 4d699bd..aaf9638 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```
- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.
```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```
- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -835,14 +835,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.
```
sudo rm -rf tmp
```
- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):
```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326..0000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
- # Check if this package is already in base-apt
- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
-
-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b687fcf..cfd617a 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 6d4d42c..a10e7cc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index d86c5b9..e46142d 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5b14cd3..57cdea9 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():
do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] = "base-apt:do_cache"
isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index c9a7dad..2dba779 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
+
+ # Check if this package is already in base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
"${KEYFILES}"
+
+ if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+ [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ # We would need two separate repository paths for that.
+ # Otherwise packages (especially the 'all' arch ones) from one
+ # distribution can influence the package versions of the other
+ # distribution.
+ bbfatal "Different host and target distributions are currently not supported." \
+ "Try it without cross-build."
+ fi
+
+ populate_base_apt
}
-python do_cache_config() {
+python do_cache() {
+ if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ return 0
+
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
filename = fetcher.localpath(key)
d.appendVar("KEYFILES", " %s" % filename)
- bb.build.exec_func('repo_config', d)
+ bb.build.exec_func('repo', d)
}
-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee9..733ba86 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
# Enable use of unsigned cached base repository
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 24/26] meta: base-apt: remove isar-apt check from population
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (22 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 23/26] base-apt: pull base-apt population to the front of the build chain Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 25/26] meta: deb-dl-dir: only export newly downloaded files Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache Vijai Kumar K
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Now that we populate just from the DL_DIR and filter out the isar-apt
packages when populating this, we can skip that test since it will never
match.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-devtools/base-apt/base-apt.bb | 4 ----
1 file changed, 4 deletions(-)
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 2dba779..cfc26ed 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -17,10 +17,6 @@ populate_base_apt() {
# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
# Check if this package is already in base-apt
ret=0
repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 25/26] meta: deb-dl-dir: only export newly downloaded files
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (23 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 24/26] meta: base-apt: remove isar-apt check from population Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 15:42 ` [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache Vijai Kumar K
25 siblings, 0 replies; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Remember the time we imported the previously downloaded files, and only
copy out new ones when exporting again. That saved pointless copies and
time.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index f969960..e65fc51 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -8,24 +8,31 @@ inherit repository
deb_dl_dir_import() {
export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
+ rm -f "${T}"/deb_dl_dir_import.stamp
+ touch "${T}"/deb_dl_dir_import.stamp
[ ! -d "${pc}" ] && return 0
+ [ ! -d "${rootfs}"/var/cache/apt/archives/ ] && return 0
flock -s "${pc}".lock -c ' \
sudo find "${pc}" -type f -iname '*\.deb' -exec \
- cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
'
}
deb_dl_dir_export() {
export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
+ export T
mkdir -p "${pc}"
flock "${pc}".lock -c ' \
- find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ find "${rootfs}"/var/cache/apt/archives/ \
+ -cnewer "${T}"/deb_dl_dir_import.stamp \
+ -type f -iname '*\.deb' |\
while read p; do
repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
continue
- sudo cp -f "${p}" "${pc}"
+ sudo cp -n "${p}" "${pc}"
done
sudo chown -R $(id -u):$(id -g) "${pc}"
'
+ rm -f "${T}"/deb_dl_dir_import.stamp
}
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-28 15:41 ` Base-apt rework v3 Vijai Kumar K
` (24 preceding siblings ...)
2020-01-28 15:42 ` [PATCH v3 25/26] meta: deb-dl-dir: only export newly downloaded files Vijai Kumar K
@ 2020-01-28 15:42 ` Vijai Kumar K
2020-01-28 16:52 ` Henning Schild
25 siblings, 1 reply; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-28 15:42 UTC (permalink / raw)
To: isar-users; +Cc: Vijai Kumar K
From: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
linux-custom overrides do_dpkg_builddeps with its own variant.
Since deb-dl-dir* is not called KBUILD_DEPENDS is not cached.
Call deb-dl-dir-import/deb-dl-dir-export to cache it.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/recipes-kernel/linux/linux-custom.inc | 2 ++
1 file changed, 2 insertions(+)
diff --git a/meta/recipes-kernel/linux/linux-custom.inc b/meta/recipes-kernel/linux/linux-custom.inc
index 17f8464..37eb3c2 100644
--- a/meta/recipes-kernel/linux/linux-custom.inc
+++ b/meta/recipes-kernel/linux/linux-custom.inc
@@ -41,6 +41,7 @@ KERNEL_LIBC_DEV_DEPLOY ?= "0"
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get update \
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
@@ -50,6 +51,7 @@ do_install_builddeps() {
apt-get install \
-y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${KBUILD_DEPENDS}
+ deb_dl_dir_export "${BUILDCHROOT_DIR}"
dpkg_undo_mounts
}
--
2.17.1
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-28 15:42 ` [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache Vijai Kumar K
@ 2020-01-28 16:52 ` Henning Schild
2020-01-28 19:45 ` vijai kumar
0 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2020-01-28 16:52 UTC (permalink / raw)
To: Vijai Kumar K; +Cc: isar-users, Vijai Kumar K
Thanks.
On Tue, 28 Jan 2020 21:12:13 +0530
Vijai Kumar K <vijaikumar.kanagarajan@gmail.com> wrote:
> From: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
>
> linux-custom overrides do_dpkg_builddeps with its own variant.
> Since deb-dl-dir* is not called KBUILD_DEPENDS is not cached.
>
> Call deb-dl-dir-import/deb-dl-dir-export to cache it.
>
> Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
> ---
> meta/recipes-kernel/linux/linux-custom.inc | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/meta/recipes-kernel/linux/linux-custom.inc
> b/meta/recipes-kernel/linux/linux-custom.inc index 17f8464..37eb3c2
> 100644 --- a/meta/recipes-kernel/linux/linux-custom.inc
> +++ b/meta/recipes-kernel/linux/linux-custom.inc
> @@ -41,6 +41,7 @@ KERNEL_LIBC_DEV_DEPLOY ?= "0"
> do_install_builddeps() {
> dpkg_do_mounts
> E="${@ isar_export_proxies(d)}"
> + deb_dl_dir_import "${BUILDCHROOT_DIR}"
> sudo -E chroot ${BUILDCHROOT_DIR} \
> apt-get update \
> -o
> Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ @@ -50,6 +51,7
> @@ do_install_builddeps() { apt-get install \
> -y -o Debug::pkgProblemResolver=yes \
> --no-install-recommends ${KBUILD_DEPENDS}
In between here you will have to do the split from p8. Otherwise if you
have post-install hooks to clear the caches the next statement will
copy nothing.
I guess it is ok to send just one patch and note that it is supposed to
sit on top of my v2.
Henning
> + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> dpkg_undo_mounts
> }
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-28 16:52 ` Henning Schild
@ 2020-01-28 19:45 ` vijai kumar
2020-01-28 21:22 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: vijai kumar @ 2020-01-28 19:45 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 1957 bytes --]
On Tuesday, January 28, 2020 at 10:22:37 PM UTC+5:30, Henning Schild wrote:
>
> Thanks.
>
> On Tue, 28 Jan 2020 21:12:13 +0530
> Vijai Kumar K <vijaikumar...@gmail.com <javascript:>> wrote:
>
> > From: Vijai Kumar K <Vijaikumar_...@mentor.com <javascript:>>
> >
> > linux-custom overrides do_dpkg_builddeps with its own variant.
> > Since deb-dl-dir* is not called KBUILD_DEPENDS is not cached.
> >
> > Call deb-dl-dir-import/deb-dl-dir-export to cache it.
> >
> > Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com <javascript:>>
> > ---
> > meta/recipes-kernel/linux/linux-custom.inc | 2 ++
> > 1 file changed, 2 insertions(+)
> >
> > diff --git a/meta/recipes-kernel/linux/linux-custom.inc
> > b/meta/recipes-kernel/linux/linux-custom.inc index 17f8464..37eb3c2
> > 100644 --- a/meta/recipes-kernel/linux/linux-custom.inc
> > +++ b/meta/recipes-kernel/linux/linux-custom.inc
> > @@ -41,6 +41,7 @@ KERNEL_LIBC_DEV_DEPLOY ?= "0"
> > do_install_builddeps() {
> > dpkg_do_mounts
> > E="${@ isar_export_proxies(d)}"
> > + deb_dl_dir_import "${BUILDCHROOT_DIR}"
> > sudo -E chroot ${BUILDCHROOT_DIR} \
> > apt-get update \
> > -o
> > Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ @@ -50,6 +51,7
> > @@ do_install_builddeps() { apt-get install \
> > -y -o Debug::pkgProblemResolver=yes \
> > --no-install-recommends ${KBUILD_DEPENDS}
>
> In between here you will have to do the split from p8. Otherwise if you
> have post-install hooks to clear the caches the next statement will
> copy nothing.
>
>
Ah, I see. Will change that.
> I guess it is ok to send just one patch and note that it is supposed to
> sit on top of my v2.
Sure.
Thanks,
Vijai Kumar K
>
>
> Henning
>
> > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > dpkg_undo_mounts
> > }
> >
>
>
[-- Attachment #1.2: Type: text/html, Size: 3522 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-28 19:45 ` vijai kumar
@ 2020-01-28 21:22 ` Henning Schild
2020-01-29 5:29 ` Vijai Kumar K
0 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2020-01-28 21:22 UTC (permalink / raw)
To: vijai kumar; +Cc: isar-users
On Tue, 28 Jan 2020 11:45:18 -0800
vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
> On Tuesday, January 28, 2020 at 10:22:37 PM UTC+5:30, Henning Schild
> wrote:
> >
> > Thanks.
> >
> > On Tue, 28 Jan 2020 21:12:13 +0530
> > Vijai Kumar K <vijaikumar...@gmail.com <javascript:>> wrote:
> >
> > > From: Vijai Kumar K <Vijaikumar_...@mentor.com <javascript:>>
> > >
> > > linux-custom overrides do_dpkg_builddeps with its own variant.
> > > Since deb-dl-dir* is not called KBUILD_DEPENDS is not cached.
> > >
> > > Call deb-dl-dir-import/deb-dl-dir-export to cache it.
> > >
> > > Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com
> > > <javascript:>> ---
> > > meta/recipes-kernel/linux/linux-custom.inc | 2 ++
> > > 1 file changed, 2 insertions(+)
> > >
> > > diff --git a/meta/recipes-kernel/linux/linux-custom.inc
> > > b/meta/recipes-kernel/linux/linux-custom.inc index
> > > 17f8464..37eb3c2 100644 ---
> > > a/meta/recipes-kernel/linux/linux-custom.inc +++
> > > b/meta/recipes-kernel/linux/linux-custom.inc @@ -41,6 +41,7 @@
> > > KERNEL_LIBC_DEV_DEPLOY ?= "0" do_install_builddeps() {
> > > dpkg_do_mounts
> > > E="${@ isar_export_proxies(d)}"
> > > + deb_dl_dir_import "${BUILDCHROOT_DIR}"
> > > sudo -E chroot ${BUILDCHROOT_DIR} \
> > > apt-get update \
> > > -o
> > > Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ @@ -50,6
> > > +51,7 @@ do_install_builddeps() { apt-get install \
> > > -y -o Debug::pkgProblemResolver=yes \
> > > --no-install-recommends
> > > ${KBUILD_DEPENDS}
> >
> > In between here you will have to do the split from p8. Otherwise if
> > you have post-install hooks to clear the caches the next statement
> > will copy nothing.
> >
> >
> Ah, I see. Will change that.
It is not likely to have such a hook for the buildchroot but we can not
rule it out for the image. Let us just follow the pattern for the
buildchroot as well.
>
> > I guess it is ok to send just one patch and note that it is
> > supposed to sit on top of my v2.
>
>
> Sure.
I could fold your improvements into the queue as regular review
feedback. Or we make it a patch on top to give you credit for finding
the problem with the custom kernel.
One thing that needs to be done as well, is including a custom-kernel
build into the ci tests. The rebuild tests not covering a custom-kernel
is the reason you ran into the issue. If you decide to send an update
of your patch on top of my patches, please make that ci change as well.
Otherwise let me know if you are ok with me taking the review comment
and doing all the required changes in a v4.
In case the maintainer is actually reading all this ... did you have
any other issues with this queue? I know you are including it into a
rather complex layer. If that was the only finding a v4 might just be
ready for a merge.
regards,
Henning
> Thanks,
> Vijai Kumar K
>
>
> >
> >
> > Henning
> >
> > > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > > dpkg_undo_mounts
> > > }
> > >
> >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-28 21:22 ` Henning Schild
@ 2020-01-29 5:29 ` Vijai Kumar K
2020-01-31 9:27 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: Vijai Kumar K @ 2020-01-29 5:29 UTC (permalink / raw)
To: Henning Schild; +Cc: vijai kumar, isar-users
On Tue, Jan 28, 2020 at 10:22:40PM +0100, Henning Schild wrote:
> On Tue, 28 Jan 2020 11:45:18 -0800
> vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
>
> > On Tuesday, January 28, 2020 at 10:22:37 PM UTC+5:30, Henning Schild
> > wrote:
> > >
> > > Thanks.
> > >
> > > On Tue, 28 Jan 2020 21:12:13 +0530
> > > Vijai Kumar K <vijaikumar...@gmail.com <javascript:>> wrote:
> > >
> > > > From: Vijai Kumar K <Vijaikumar_...@mentor.com <javascript:>>
> > > >
> > > > linux-custom overrides do_dpkg_builddeps with its own variant.
> > > > Since deb-dl-dir* is not called KBUILD_DEPENDS is not cached.
> > > >
> > > > Call deb-dl-dir-import/deb-dl-dir-export to cache it.
> > > >
> > > > Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com
> > > > <javascript:>> ---
> > > > meta/recipes-kernel/linux/linux-custom.inc | 2 ++
> > > > 1 file changed, 2 insertions(+)
> > > >
> > > > diff --git a/meta/recipes-kernel/linux/linux-custom.inc
> > > > b/meta/recipes-kernel/linux/linux-custom.inc index
> > > > 17f8464..37eb3c2 100644 ---
> > > > a/meta/recipes-kernel/linux/linux-custom.inc +++
> > > > b/meta/recipes-kernel/linux/linux-custom.inc @@ -41,6 +41,7 @@
> > > > KERNEL_LIBC_DEV_DEPLOY ?= "0" do_install_builddeps() {
> > > > dpkg_do_mounts
> > > > E="${@ isar_export_proxies(d)}"
> > > > + deb_dl_dir_import "${BUILDCHROOT_DIR}"
> > > > sudo -E chroot ${BUILDCHROOT_DIR} \
> > > > apt-get update \
> > > > -o
> > > > Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ @@ -50,6
> > > > +51,7 @@ do_install_builddeps() { apt-get install \
> > > > -y -o Debug::pkgProblemResolver=yes \
> > > > --no-install-recommends
> > > > ${KBUILD_DEPENDS}
> > >
> > > In between here you will have to do the split from p8. Otherwise if
> > > you have post-install hooks to clear the caches the next statement
> > > will copy nothing.
> > >
> > >
> > Ah, I see. Will change that.
>
> It is not likely to have such a hook for the buildchroot but we can not
> rule it out for the image. Let us just follow the pattern for the
> buildchroot as well.
>
> >
> > > I guess it is ok to send just one patch and note that it is
> > > supposed to sit on top of my v2.
> >
> >
> > Sure.
>
> I could fold your improvements into the queue as regular review
> feedback. Or we make it a patch on top to give you credit for finding
> the problem with the custom kernel.
>
> One thing that needs to be done as well, is including a custom-kernel
> build into the ci tests. The rebuild tests not covering a custom-kernel
> is the reason you ran into the issue. If you decide to send an update
> of your patch on top of my patches, please make that ci change as well.
> Otherwise let me know if you are ok with me taking the review comment
> and doing all the required changes in a v4.
No problem. Please go ahead.
Thanks,
Vijai Kumar K
>
> In case the maintainer is actually reading all this ... did you have
> any other issues with this queue? I know you are including it into a
> rather complex layer. If that was the only finding a v4 might just be
> ready for a merge.
>
> regards,
> Henning
>
> > Thanks,
> > Vijai Kumar K
> >
> >
> > >
> > >
> > > Henning
> > >
> > > > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > > > dpkg_undo_mounts
> > > > }
> > > >
> > >
> > >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCH v3 26/26] meta/recipes-kernel: Export KBUILD_DEPENDS to cache
2020-01-29 5:29 ` Vijai Kumar K
@ 2020-01-31 9:27 ` Henning Schild
0 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2020-01-31 9:27 UTC (permalink / raw)
To: Vijai Kumar K; +Cc: isar-users
On Wed, 29 Jan 2020 10:59:18 +0530
Vijai Kumar K <vijaikumar.kanagarajan@gmail.com> wrote:
> On Tue, Jan 28, 2020 at 10:22:40PM +0100, Henning Schild wrote:
> > On Tue, 28 Jan 2020 11:45:18 -0800
> > vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
> >
> > > On Tuesday, January 28, 2020 at 10:22:37 PM UTC+5:30, Henning
> > > Schild wrote:
> > > >
> > > > Thanks.
> > > >
> > > > On Tue, 28 Jan 2020 21:12:13 +0530
> > > > Vijai Kumar K <vijaikumar...@gmail.com <javascript:>> wrote:
> > > >
> > > > > From: Vijai Kumar K <Vijaikumar_...@mentor.com <javascript:>>
> > > > >
> > > > > linux-custom overrides do_dpkg_builddeps with its own
> > > > > variant. Since deb-dl-dir* is not called KBUILD_DEPENDS is
> > > > > not cached.
> > > > >
> > > > > Call deb-dl-dir-import/deb-dl-dir-export to cache it.
> > > > >
> > > > > Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com
> > > > > <javascript:>> ---
> > > > > meta/recipes-kernel/linux/linux-custom.inc | 2 ++
> > > > > 1 file changed, 2 insertions(+)
> > > > >
> > > > > diff --git a/meta/recipes-kernel/linux/linux-custom.inc
> > > > > b/meta/recipes-kernel/linux/linux-custom.inc index
> > > > > 17f8464..37eb3c2 100644 ---
> > > > > a/meta/recipes-kernel/linux/linux-custom.inc +++
> > > > > b/meta/recipes-kernel/linux/linux-custom.inc @@ -41,6 +41,7 @@
> > > > > KERNEL_LIBC_DEV_DEPLOY ?= "0" do_install_builddeps() {
> > > > > dpkg_do_mounts
> > > > > E="${@ isar_export_proxies(d)}"
> > > > > + deb_dl_dir_import "${BUILDCHROOT_DIR}"
> > > > > sudo -E chroot ${BUILDCHROOT_DIR} \
> > > > > apt-get update \
> > > > > -o
> > > > > Dir::Etc::SourceList="sources.list.d/isar-apt.list" \ @@ -50,6
> > > > > +51,7 @@ do_install_builddeps() { apt-get install \
> > > > > -y -o Debug::pkgProblemResolver=yes
> > > > > \ --no-install-recommends
> > > > > ${KBUILD_DEPENDS}
> > > >
> > > > In between here you will have to do the split from p8.
> > > > Otherwise if you have post-install hooks to clear the caches
> > > > the next statement will copy nothing.
> > > >
> > > >
> > > Ah, I see. Will change that.
> >
> > It is not likely to have such a hook for the buildchroot but we can
> > not rule it out for the image. Let us just follow the pattern for
> > the buildchroot as well.
> >
> > >
> > > > I guess it is ok to send just one patch and note that it is
> > > > supposed to sit on top of my v2.
> > >
> > >
> > > Sure.
> >
> > I could fold your improvements into the queue as regular review
> > feedback. Or we make it a patch on top to give you credit for
> > finding the problem with the custom kernel.
> >
> > One thing that needs to be done as well, is including a
> > custom-kernel build into the ci tests. The rebuild tests not
> > covering a custom-kernel is the reason you ran into the issue. If
> > you decide to send an update of your patch on top of my patches,
> > please make that ci change as well. Otherwise let me know if you
> > are ok with me taking the review comment and doing all the required
> > changes in a v4.
>
> No problem. Please go ahead.
Turns out that is not needed anymore when basing the patches on current
next. I will send an update of the series rebased ... and merged ...,
it will also include a CI test for custom kernel in the cached case.
Henning
> Thanks,
> Vijai Kumar K
>
> >
> > In case the maintainer is actually reading all this ... did you have
> > any other issues with this queue? I know you are including it into a
> > rather complex layer. If that was the only finding a v4 might just
> > be ready for a merge.
> >
> > regards,
> > Henning
> >
> > > Thanks,
> > > Vijai Kumar K
> > >
> > >
> > > >
> > > >
> > > > Henning
> > > >
> > > > > + deb_dl_dir_export "${BUILDCHROOT_DIR}"
> > > > > dpkg_undo_mounts
> > > > > }
> > > > >
> > > >
> > > >
> > >
> >
^ permalink raw reply [flat|nested] 68+ messages in thread
* [PATCHv2 10/25] meta: import DL_DIR debs before apt-get download steps
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (8 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 11/25] meta: include DL_DIR deb import/export into bootstrap Henning Schild
` (15 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
If we ever downloaded a file there is no need to do that again. We can
not only use that DL_DIR to later construct a repository but also to
speed up our rebuilds.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 10 ++++++++++
meta/classes/dpkg-gbp.bbclass | 1 +
meta/classes/dpkg.bbclass | 1 +
meta/classes/image-tools-extension.bbclass | 1 +
meta/classes/rootfs.bbclass | 6 ++++++
5 files changed, 19 insertions(+)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index e41e981..ab4b1f0 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,6 +3,16 @@
#
# SPDX-License-Identifier: MIT
+deb_dl_dir_import() {
+ export pc="${DEBDIR}/${DISTRO}"
+ export rootfs="${1}"
+ [ ! -d "${pc}" ] && return 0
+ flock -s "${pc}".lock -c ' \
+ sudo find "${pc}" -type f -iname '*\.deb' -exec \
+ cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ '
+}
+
deb_dl_dir_export() {
export pc="${DEBDIR}/${DISTRO}"
export rootfs="${1}"
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index 7e908eb..afa1e19 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,6 +12,7 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
do_install_builddeps_append() {
dpkg_do_mounts
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index 7e2c86d..585365e 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -7,6 +7,7 @@ inherit dpkg-base
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import "${BUILDCHROOT_DIR}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${DISTRO_ARCH} --download-only
deb_dl_dir_export "${BUILDCHROOT_DIR}"
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index ec67d94..6590ee7 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,6 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts
E="${@ isar_export_proxies(d)}"
+ deb_dl_dir_import ${BUILDCHROOT_DIR}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get update \
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index a6b6110..eac553c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -113,6 +113,12 @@ rootfs_install_resolvconf() {
fi
}
+ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
+rootfs_import_package_cache[weight] = "5"
+rootfs_import_package_cache() {
+ deb_dl_dir_import ${ROOTFSDIR}
+}
+
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
rootfs_install_pkgs_download[weight] = "600"
rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 11/25] meta: include DL_DIR deb import/export into bootstrap
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (9 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 10/25] meta: import DL_DIR debs before apt-get download steps Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 12/25] base-apt: populate from DEBDIR as well Henning Schild
` (14 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The bootstrap step also contains downloading of debs, so it should be
surrounded with an import/export as well.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index cfcf7b8..5b14cd3 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -28,6 +28,8 @@ DISTRO_BOOTSTRAP_BASE_PACKAGES_append_gnupg = ",gnupg"
DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = ",apt-transport-https,ca-certificates"
HOST_DISTRO_APT_SOURCES += "conf/distro/${HOST_DISTRO}.list"
+inherit deb-dl-dir
+
python () {
distro_bootstrap_keys = (d.getVar("DISTRO_BOOTSTRAP_KEYS") or "").split()
@@ -219,6 +221,7 @@ do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
isar_bootstrap() {
+ deb_dl_dir_import "${ROOTFSDIR}"
IS_HOST=""
while true; do
case "$1" in
@@ -335,6 +338,7 @@ isar_bootstrap() {
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
fi
EOSUDO
+ deb_dl_dir_export "${ROOTFSDIR}"
}
CLEANFUNCS = "clean_deploy"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 12/25] base-apt: populate from DEBDIR as well
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (10 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 11/25] meta: include DL_DIR deb import/export into bootstrap Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:49 ` [PATCHv2 13/25] base-apt: drop the "apt_cache" feature Henning Schild
` (13 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We have all the .debs cached in DEBDIR now, take them from there because
the rootfss might not actually have all of them.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index f8b9ba3..07e9e7d 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -33,6 +33,10 @@ do_cache_base_repo() {
populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
fi
+ if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
+ populate_base_apt "${DEBDIR}"/"${DISTRO}"
+ fi
+
if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
fi
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 13/25] base-apt: drop the "apt_cache" feature
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (11 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 12/25] base-apt: populate from DEBDIR as well Henning Schild
@ 2019-12-16 11:49 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 14/25] base-apt: do not copy debs directly out of rootfss anymore Henning Schild
` (12 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:49 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
We now have all debs in DEBDIR anyways, no need to conditionally copy
them out of the rootfss we create.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 4 ----
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 10 ----------
3 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 07e9e7d..19c9c59 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,10 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d ${WORKDIR}/apt_cache ]; then
- populate_base_apt ${WORKDIR}/apt_cache
- fi
-
if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
fi
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 725bc04..b687fcf 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -60,7 +60,7 @@ image_do_mounts() {
}
ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "copy-package-cache clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index eac553c..6d4d42c 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -9,7 +9,6 @@ ROOTFS_PACKAGES ?= ""
# Features of the rootfs creation:
# available features are:
-# 'copy-package-cache' - copy the package cache ${WORKDIR}/apt_cache
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
# 'finalize-rootfs' - delete files needed to chroot into the rootfs
@@ -186,15 +185,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack
-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'copy-package-cache', 'rootfs_postprocess_copy_package_cache', '', d)}"
-rootfs_postprocess_copy_package_cache() {
- mkdir -p '${WORKDIR}/apt_cache'
- sudo find '${ROOTFSDIR}/var/cache/apt/archives' \
- -maxdepth 1 -name '*.deb' -execdir /bin/mv -t '${WORKDIR}/apt_cache' '{}' '+'
- me="$(id -u):$(id -g)"
- sudo chown -R "$me" '${WORKDIR}/apt_cache'
-}
-
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {
sudo -E chroot '${ROOTFSDIR}' \
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 14/25] base-apt: do not copy debs directly out of rootfss anymore
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (12 preceding siblings ...)
2019-12-16 11:49 ` [PATCHv2 13/25] base-apt: drop the "apt_cache" feature Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 15/25] base-apt: rework base-apt population Henning Schild
` (11 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The earlier commits make sure we already have all .debs in the repo, no
need to get them from these locations. In fact these locations are not
reliable anyways, see hook in "isar-disable-apt-cache".
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 8 --------
1 file changed, 8 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 19c9c59..cff2a8b 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,14 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d ${BUILDCHROOT_HOST_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_HOST_DIR}/var/cache/apt
- fi
-
- if [ -d ${BUILDCHROOT_TARGET_DIR}/var/cache/apt ]; then
- populate_base_apt ${BUILDCHROOT_TARGET_DIR}/var/cache/apt
- fi
-
if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
populate_base_apt "${DEBDIR}"/"${DISTRO}"
fi
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 15/25] base-apt: rework base-apt population
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (13 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 14/25] base-apt: do not copy debs directly out of rootfss anymore Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 16/25] base-apt: move class "base-apt-helper" into only user Henning Schild
` (10 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Just call that function once and make it find the debs and dscs in one
run.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 6 ++----
meta/classes/image-cache-extension.bbclass | 8 +-------
2 files changed, 3 insertions(+), 11 deletions(-)
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
index db25adc..18d43aa 100644
--- a/meta/classes/base-apt-helper.bbclass
+++ b/meta/classes/base-apt-helper.bbclass
@@ -17,9 +17,7 @@ compare_pkg_md5sums() {
}
populate_base_apt() {
- search_dir=$1
-
- find $search_dir -name '*.deb' | while read package; do
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
# use search by filename to check if package is already in repo. In
# addition, md5sums are compared to ensure that the package is the
@@ -53,7 +51,7 @@ populate_base_apt() {
"${package}"
done
- find $search_dir -name '*.dsc' | while read package; do
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index cff2a8b..7945e8b 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -21,12 +21,6 @@ do_cache_base_repo() {
"Try it without cross-build."
fi
- if [ -d "${DEBDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBDIR}"/"${DISTRO}"
- fi
-
- if [ -d "${DEBSRCDIR}"/"${DISTRO}" ]; then
- populate_base_apt "${DEBSRCDIR}"/"${DISTRO}"
- fi
+ populate_base_apt
}
addtask cache_base_repo after do_rootfs do_install_imager_deps
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 16/25] base-apt: move class "base-apt-helper" into only user
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (14 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 15/25] base-apt: rework base-apt population Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 17/25] CI: include "isar-disable-apt-cache" into all CI images Henning Schild
` (9 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
There is only one place we use that code, move it there instead of
having an extra class.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/base-apt-helper.bbclass | 60 ----------------------
meta/classes/image-cache-extension.bbclass | 55 +++++++++++++++++++-
2 files changed, 54 insertions(+), 61 deletions(-)
delete mode 100644 meta/classes/base-apt-helper.bbclass
diff --git a/meta/classes/base-apt-helper.bbclass b/meta/classes/base-apt-helper.bbclass
deleted file mode 100644
index 18d43aa..0000000
--- a/meta/classes/base-apt-helper.bbclass
+++ /dev/null
@@ -1,60 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) 2018 ilbers GmbH
-# Copyright (C) 2019 Siemens AG
-#
-# SPDX-License-Identifier: MIT
-
-inherit repository
-
-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
-
- # Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index 7945e8b..c3ee7b8 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -5,7 +5,60 @@
#
# This class extends the image.bbclass to supply the creation of cache repositories
-inherit base-apt-helper
+inherit repository
+
+compare_pkg_md5sums() {
+ pkg1=$1
+ pkg2=$2
+
+ md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
+ md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
+
+ [ "$md1" = "$md2" ]
+}
+
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
+
+ # Check if this package is taken from Isar-apt, if so - ignore it.
+ base_name=${package##*/}
+ isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
+ if [ -n "$isar_apt_p" ]; then
+ # Check if MD5 sums are identical. This helps to avoid the case
+ # when packages is overridden from another repo.
+ compare_pkg_md5sums "$package" "$isar_apt_p" && continue
+ fi
+
+ # Check if this package is already in base-apt
+ base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
+ if [ -n "$base_apt_p" ]; then
+ compare_pkg_md5sums "$package" "$base_apt_p" && continue
+
+ # md5sum differs, so remove the package from base-apt
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
do_cache_base_repo[depends] = "base-apt:do_cache_config"
do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 17/25] CI: include "isar-disable-apt-cache" into all CI images
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (15 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 16/25] base-apt: move class "base-apt-helper" into only user Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 18/25] CI: include "cowsay" into default build to test dpkg-gbp Henning Schild
` (8 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
The base-apt does not rely on the rootfs-cache anymore, so we can test
our cleanup package in all images.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 4 +---
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 0e55da1..dee1250 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,9 +166,7 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello"
-## "cache_base_repo" relies on the cache ...
-#IMAGE_INSTALL += "isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index f50c9e4..bfaf1f6 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi
-sed -i -e 's/#IMAGE_INSTALL += "isar-disable-apt-cache"/IMAGE_INSTALL += "isar-disable-apt-cache"/g' conf/local.conf
-
echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
# Start cross build for the defined set of configurations
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 18/25] CI: include "cowsay" into default build to test dpkg-gbp
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (16 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 17/25] CI: include "isar-disable-apt-cache" into all CI images Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 19/25] CI: set BB_NO_NETWORK for cached rebuild Henning Schild
` (7 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta-isar/conf/local.conf.sample | 2 +-
scripts/ci_build.sh | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index dee1250..9237bb3 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -166,7 +166,7 @@ CONF_VERSION = "1"
#
# The default list of extra packages to be installed.
-IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache"
+IMAGE_INSTALL = "hello-isar example-raw example-module-${KERNEL_NAME} enable-fsck isar-exclude-docs samefile hello isar-disable-apt-cache cowsay"
#
# Enable cross-compilation support
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index bfaf1f6..d440c24 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -168,8 +168,6 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
fi
-echo 'IMAGE_INSTALL += "cowsay"' >> conf/local.conf
-
# Start cross build for the defined set of configurations
sed -i -e 's/ISAR_CROSS_COMPILE ?= "0"/ISAR_CROSS_COMPILE ?= "1"/g' conf/local.conf
bitbake $BB_ARGS $CROSS_TARGETS_SET
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 19/25] CI: set BB_NO_NETWORK for cached rebuild
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (17 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 18/25] CI: include "cowsay" into default build to test dpkg-gbp Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 20/25] meta: repository: implement repo_contains_package and use it in base_apt Henning Schild
` (6 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Also add it to our sample config and comment on its use in conjuction
with base-apt.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
doc/user_manual.md | 1 +
meta-isar/conf/local.conf.sample | 4 ++++
scripts/ci_build.sh | 4 ++++
3 files changed, 9 insertions(+)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index c9062d6..4d699bd 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -833,6 +833,7 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
```
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#BB_NO_NETWORK ?= "1"
```
- Remove build artifacts to use only local base-apt:
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index 9237bb3..8355a40 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -176,6 +176,10 @@ ISAR_CROSS_COMPILE ?= "0"
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
+#
+# You probably want to uncomment this as well to make sure the build
+# does not access the network
+#BB_NO_NETWORK ?= "1"
# Set root password to 'root'
# Password was encrypted using following command:
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index d440c24..7da3ee9 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -149,11 +149,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of signed cached base repository
sudo rm -rf tmp
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
# Enable use of unsigned cached base repository
@@ -161,11 +163,13 @@ if [ -n "$REPRO_BUILD" ]; then
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^#BB_NO_NETWORK/BB_NO_NETWORK/g' conf/local.conf
bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
# Cleanup and disable use of unsigned cached base repository
sudo rm -rf tmp
sed -i -e 's/ISAR_USE_CACHED_BASE_REPO ?= "1"/#ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
+ sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
fi
# Start cross build for the defined set of configurations
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 20/25] meta: repository: implement repo_contains_package and use it in base_apt
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (18 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 19/25] CI: set BB_NO_NETWORK for cached rebuild Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 21/25] meta: repository: simplify the matching of packages Henning Schild
` (5 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Make the lookup and the checksum comparison one function of a
repository. That cleans things up and allows for easier reuse.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/image-cache-extension.bbclass | 31 ++++++----------------
meta/classes/repository.bbclass | 19 +++++++++++++
2 files changed, 27 insertions(+), 23 deletions(-)
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
index c3ee7b8..4123326 100644
--- a/meta/classes/image-cache-extension.bbclass
+++ b/meta/classes/image-cache-extension.bbclass
@@ -7,16 +7,6 @@
inherit repository
-compare_pkg_md5sums() {
- pkg1=$1
- pkg2=$2
-
- md1=$(md5sum $pkg1 | cut -d ' ' -f 1)
- md2=$(md5sum $pkg2 | cut -d ' ' -f 1)
-
- [ "$md1" = "$md2" ]
-}
-
populate_base_apt() {
find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
# NOTE: due to packages stored by reprepro are not modified, we can
@@ -25,21 +15,16 @@ populate_base_apt() {
# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.
- # Check if this package is taken from Isar-apt, if so - ignore it.
- base_name=${package##*/}
- isar_apt_p=$(find ${REPO_ISAR_DIR}/${DISTRO} -name $base_name)
- if [ -n "$isar_apt_p" ]; then
- # Check if MD5 sums are identical. This helps to avoid the case
- # when packages is overridden from another repo.
- compare_pkg_md5sums "$package" "$isar_apt_p" && continue
- fi
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
# Check if this package is already in base-apt
- base_apt_p=$(find ${REPO_BASE_DIR}/${BASE_DISTRO} -name $base_name)
- if [ -n "$base_apt_p" ]; then
- compare_pkg_md5sums "$package" "$base_apt_p" && continue
-
- # md5sum differs, so remove the package from base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index dc4c1dc..e7acf59 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -81,3 +81,22 @@ repo_del_package() {
remove "${codename}" \
"${p}"
}
+
+repo_contains_package() {
+ local dir="$1"
+ local file="$2"
+ local package
+
+ package=$(find ${dir} -name ${file##*/})
+ if [ -n "$package" ]; then
+ local md1=$(md5sum "$package" | cut -d ' ' -f 1)
+ local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
+
+ # yes
+ [ "${md1}" = "${md2}" ] && return 0
+ # yes but not the exact same file
+ return 1
+ fi
+ # no
+ return 2
+}
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 21/25] meta: repository: simplify the matching of packages
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (19 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 20/25] meta: repository: implement repo_contains_package and use it in base_apt Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 22/25] meta: deb-dl-dir: do not cache debs from isar-apt Henning Schild
` (4 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Just call cmp once instead of comparing the md5 sums.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/repository.bbclass | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index e7acf59..de1fa5e 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -89,11 +89,9 @@ repo_contains_package() {
package=$(find ${dir} -name ${file##*/})
if [ -n "$package" ]; then
- local md1=$(md5sum "$package" | cut -d ' ' -f 1)
- local md2=$(sudo md5sum "$file" | cut -d ' ' -f 1)
-
# yes
- [ "${md1}" = "${md2}" ] && return 0
+ cmp --silent "$package" "$file" && return 0
+
# yes but not the exact same file
return 1
fi
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 22/25] meta: deb-dl-dir: do not cache debs from isar-apt
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (20 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 21/25] meta: repository: simplify the matching of packages Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain Henning Schild
` (3 subsequent siblings)
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Packages from isar-apt are not downloaded from the outside and should
not be cached.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index ab4b1f0..f969960 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -3,8 +3,10 @@
#
# SPDX-License-Identifier: MIT
+inherit repository
+
deb_dl_dir_import() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
[ ! -d "${pc}" ] && return 0
flock -s "${pc}".lock -c ' \
@@ -14,12 +16,16 @@ deb_dl_dir_import() {
}
deb_dl_dir_export() {
- export pc="${DEBDIR}/${DISTRO}"
+ export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
mkdir -p "${pc}"
flock "${pc}".lock -c ' \
- sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' \
- -exec cp -f '{}' "${pc}" \;
+ find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ while read p; do
+ repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
+ continue
+ sudo cp -f "${p}" "${pc}"
+ done
sudo chown -R $(id -u):$(id -g) "${pc}"
'
}
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (21 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 22/25] meta: deb-dl-dir: do not cache debs from isar-apt Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2020-01-21 8:55 ` vijai kumar
2019-12-16 11:50 ` [PATCHv2 24/25] meta: base-apt: remove isar-apt check from population Henning Schild
` (2 subsequent siblings)
25 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Users had to call a special task before switching to using base-apt.
With this commit you flip one switch and the repo will be created as one
of the first steps.
The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
"processed" files and not just a download-cache.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
doc/user_manual.md | 8 +--
meta/classes/image-cache-extension.bbclass | 64 ------------------
meta/classes/image.bbclass | 1 -
meta/classes/rootfs.bbclass | 2 +-
meta/conf/bitbake.conf | 4 +-
.../isar-bootstrap/isar-bootstrap.inc | 1 +
meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
scripts/ci_build.sh | 4 +-
8 files changed, 69 insertions(+), 82 deletions(-)
delete mode 100644 meta/classes/image-cache-extension.bbclass
diff --git a/doc/user_manual.md b/doc/user_manual.md
index 4d699bd..aaf9638 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.:
BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
```
- - Trigger creation of local apt caching Debian packages during image generation.
+ - Trigger the download and caching of all required files by doing a warm-up build.
```
-bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
+bitbake mc:qemuarm-buster:isar-image-base
```
- Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
@@ -835,14 +835,14 @@ bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
#ISAR_USE_CACHED_BASE_REPO ?= "1"
#BB_NO_NETWORK ?= "1"
```
- - Remove build artifacts to use only local base-apt:
+ - Remove build artifacts to use only local base-apt, in fact toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is just the way to be extra sure that only the download cache is used.
```
sudo rm -rf tmp
```
- - Trigger again generation of image (now using local caching repo):
+ - Trigger the generation of your image again (now a local repo will be created out of the download cache from the last run):
```
bitbake mc:qemuarm-buster:isar-image-base
diff --git a/meta/classes/image-cache-extension.bbclass b/meta/classes/image-cache-extension.bbclass
deleted file mode 100644
index 4123326..0000000
--- a/meta/classes/image-cache-extension.bbclass
+++ /dev/null
@@ -1,64 +0,0 @@
-# This software is a part of ISAR.
-# Copyright (C) Siemens AG, 2019
-#
-# SPDX-License-Identifier: MIT
-#
-# This class extends the image.bbclass to supply the creation of cache repositories
-
-inherit repository
-
-populate_base_apt() {
- find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
- # NOTE: due to packages stored by reprepro are not modified, we can
- # use search by filename to check if package is already in repo. In
- # addition, md5sums are compared to ensure that the package is the
- # same and should not be overwritten. This method is easier and more
- # robust than querying reprepro by name.
-
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
- # Check if this package is already in base-apt
- ret=0
- repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
- ret=$?
- [ "${ret}" = "0" ] && continue
- if [ "${ret}" = "1" ]; then
- repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${base_apt_p}"
- fi
-
- repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-
- find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
- repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
- "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
- "${BASE_DISTRO_CODENAME}" \
- "${package}"
- done
-}
-
-do_cache_base_repo[depends] = "base-apt:do_cache_config"
-do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
-do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
-do_cache_base_repo() {
- if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
- [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
- # We would need two separate repository paths for that.
- # Otherwise packages (especially the 'all' arch ones) from one
- # distribution can influence the package versions of the other
- # distribution.
- bbfatal "Different host and target distributions are currently not supported." \
- "Try it without cross-build."
- fi
-
- populate_base_apt
-}
-addtask cache_base_repo after do_rootfs do_install_imager_deps
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b687fcf..cfd617a 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
inherit rootfs
inherit image-sdk-extension
-inherit image-cache-extension
inherit image-tools-extension
inherit image-postproc-extension
inherit image-locales-extension
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 6d4d42c..a10e7cc 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND}"
-do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config"
+do_rootfs_install[depends] = "isar-bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
do_rootfs_install[deptask] = "do_deploy_deb"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or "").split()
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index d86c5b9..e46142d 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
# Base apt repository paths
-REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
-REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
+REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
+REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 5b14cd3..57cdea9 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -219,6 +219,7 @@ def get_host_release():
do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
+do_bootstrap[depends] = "base-apt:do_cache"
isar_bootstrap() {
deb_dl_dir_import "${ROOTFSDIR}"
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index c9a7dad..2dba779 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -9,26 +9,77 @@ inherit repository
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
-do_cache_config[stamp-extra-info] = "${DISTRO}"
-do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+populate_base_apt() {
+ find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
+ # NOTE: due to packages stored by reprepro are not modified, we can
+ # use search by filename to check if package is already in repo. In
+ # addition, md5sums are compared to ensure that the package is the
+ # same and should not be overwritten. This method is easier and more
+ # robust than querying reprepro by name.
-# Generate reprepro config for current distro if it doesn't exist. Once it's
-# generated, this task should do nothing.
-repo_config() {
+ # Check if this package is taken from Isar-apt, if so - ingore it.
+ repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
+ continue
+
+ # Check if this package is already in base-apt
+ ret=0
+ repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
+ ret=$?
+ [ "${ret}" = "0" ] && continue
+ if [ "${ret}" = "1" ]; then
+ repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${base_apt_p}"
+ fi
+
+ repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+
+ find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package; do
+ repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
+ "${BASE_DISTRO_CODENAME}" \
+ "${package}"
+ done
+}
+
+do_cache[stamp-extra-info] = "${DISTRO}"
+do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
+
+repo() {
repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
"${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
"${BASE_DISTRO_CODENAME}" \
"${KEYFILES}"
+
+ if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
+ [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
+ # We would need two separate repository paths for that.
+ # Otherwise packages (especially the 'all' arch ones) from one
+ # distribution can influence the package versions of the other
+ # distribution.
+ bbfatal "Different host and target distributions are currently not supported." \
+ "Try it without cross-build."
+ fi
+
+ populate_base_apt
}
-python do_cache_config() {
+python do_cache() {
+ if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
+ return 0
+
for key in d.getVar('BASE_REPO_KEY').split():
d.appendVar("SRC_URI", " %s" % key)
fetcher = bb.fetch2.Fetch([key], d)
filename = fetcher.localpath(key)
d.appendVar("KEYFILES", " %s" % filename)
- bb.build.exec_func('repo_config', d)
+ bb.build.exec_func('repo', d)
}
-addtask cache_config after do_unpack before do_build
+addtask cache after do_unpack before do_build
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index 7da3ee9..733ba86 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
# Enable use of signed cached base repository
echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >> conf/local.conf
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
+ bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
@@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
# Enable use of unsigned cached base repository
- bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
+ bitbake $BB_ARGS $REPRO_TARGETS_SET
while [ -e bitbake.sock ]; do sleep 1; done
sudo rm -rf tmp
sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?= "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain
2019-12-16 11:50 ` [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain Henning Schild
@ 2020-01-21 8:55 ` vijai kumar
2020-01-21 9:54 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: vijai kumar @ 2020-01-21 8:55 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 13687 bytes --]
Hi Henning,
Can't we have the base-apt repo generation towards the end of the build
chain?
Right now if you need to generate a base-apt then you need to build twice.
Once without
ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO set.
What if we could just generate the repo towards the end nevertheless and
let user decide
whether to use the repo in the subsequent build by setting
ISAR_USE_CACHED_BASE_REPO?
Thanks,
Vijai Kumar K
On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning Schild wrote:
>
> From: Henning Schild <hennin...@siemens.com <javascript:>>
>
> Users had to call a special task before switching to using base-apt.
> With this commit you flip one switch and the repo will be created as one
> of the first steps.
>
> The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
> "processed" files and not just a download-cache.
>
> Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> ---
> doc/user_manual.md | 8 +--
> meta/classes/image-cache-extension.bbclass | 64 ------------------
> meta/classes/image.bbclass | 1 -
> meta/classes/rootfs.bbclass | 2 +-
> meta/conf/bitbake.conf | 4 +-
> .../isar-bootstrap/isar-bootstrap.inc | 1 +
> meta/recipes-devtools/base-apt/base-apt.bb | 67 ++++++++++++++++---
> scripts/ci_build.sh | 4 +-
> 8 files changed, 69 insertions(+), 82 deletions(-)
> delete mode 100644 meta/classes/image-cache-extension.bbclass
>
> diff --git a/doc/user_manual.md b/doc/user_manual.md
> index 4d699bd..aaf9638 100644
> --- a/doc/user_manual.md
> +++ b/doc/user_manual.md
> @@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`, e.g.:
> BASE_REPO_KEY = "file://<absolute_path_to_your_pub_key_file>"'
> ```
>
> - - Trigger creation of local apt caching Debian packages during image
> generation.
> + - Trigger the download and caching of all required files by doing a
> warm-up build.
>
> ```
> -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
> +bitbake mc:qemuarm-buster:isar-image-base
> ```
>
> - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
> @@ -835,14 +835,14 @@ bitbake -c cache_base_repo
> mc:qemuarm-buster:isar-image-base
> #ISAR_USE_CACHED_BASE_REPO ?= "1"
> #BB_NO_NETWORK ?= "1"
> ```
> - - Remove build artifacts to use only local base-apt:
> + - Remove build artifacts to use only local base-apt, in fact toggling
> ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as well. This is
> just the way to be extra sure that only the download cache is used.
>
> ```
> sudo rm -rf tmp
>
> ```
>
> - - Trigger again generation of image (now using local caching repo):
> + - Trigger the generation of your image again (now a local repo will be
> created out of the download cache from the last run):
>
> ```
> bitbake mc:qemuarm-buster:isar-image-base
> diff --git a/meta/classes/image-cache-extension.bbclass
> b/meta/classes/image-cache-extension.bbclass
> deleted file mode 100644
> index 4123326..0000000
> --- a/meta/classes/image-cache-extension.bbclass
> +++ /dev/null
> @@ -1,64 +0,0 @@
> -# This software is a part of ISAR.
> -# Copyright (C) Siemens AG, 2019
> -#
> -# SPDX-License-Identifier: MIT
> -#
> -# This class extends the image.bbclass to supply the creation of cache
> repositories
> -
> -inherit repository
> -
> -populate_base_apt() {
> - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
> - # NOTE: due to packages stored by reprepro are not modified, we
> can
> - # use search by filename to check if package is already in repo.
> In
> - # addition, md5sums are compared to ensure that the package is
> the
> - # same and should not be overwritten. This method is easier and
> more
> - # robust than querying reprepro by name.
> -
> - # Check if this package is taken from Isar-apt, if so - ingore
> it.
> - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}"
> && \
> - continue
> -
> - # Check if this package is already in base-apt
> - ret=0
> - repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> "${package}" ||
> - ret=$?
> - [ "${ret}" = "0" ] && continue
> - if [ "${ret}" = "1" ]; then
> - repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> - "${BASE_DISTRO_CODENAME}" \
> - "${base_apt_p}"
> - fi
> -
> - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> - "${BASE_DISTRO_CODENAME}" \
> - "${package}"
> - done
> -
> - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package;
> do
> - repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> - "${BASE_DISTRO_CODENAME}" \
> - "${package}"
> - done
> -}
> -
> -do_cache_base_repo[depends] = "base-apt:do_cache_config"
> -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
> -do_cache_base_repo() {
> - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> - # We would need two separate repository paths for that.
> - # Otherwise packages (especially the 'all' arch ones) from one
> - # distribution can influence the package versions of the other
> - # distribution.
> - bbfatal "Different host and target distributions are currently
> not supported." \
> - "Try it without cross-build."
> - fi
> -
> - populate_base_apt
> -}
> -addtask cache_base_repo after do_rootfs do_install_imager_deps
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index b687fcf..cfd617a 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
>
> inherit rootfs
> inherit image-sdk-extension
> -inherit image-cache-extension
> inherit image-tools-extension
> inherit image-postproc-extension
> inherit image-locales-extension
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
> index 6d4d42c..a10e7cc 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
>
> do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
> do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
> ${ROOTFS_INSTALL_COMMAND}"
> -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build
> isar-apt:do_cache_config"
> +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build
> isar-apt:do_cache_config base-apt:do_cache"
> do_rootfs_install[deptask] = "do_deploy_deb"
> python do_rootfs_install() {
> configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True) or
> "").split()
> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> index d86c5b9..e46142d 100644
> --- a/meta/conf/bitbake.conf
> +++ b/meta/conf/bitbake.conf
> @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
> THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
>
> # Base apt repository paths
> -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
> -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
> +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
> +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
>
> BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH
> BBSERVER DL_DIR \
> SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL
> TERM \
> diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> index 5b14cd3..57cdea9 100644
> --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> @@ -219,6 +219,7 @@ def get_host_release():
>
> do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> +do_bootstrap[depends] = "base-apt:do_cache"
>
> isar_bootstrap() {
> deb_dl_dir_import "${ROOTFSDIR}"
> diff --git a/meta/recipes-devtools/base-apt/base-apt.bb
> b/meta/recipes-devtools/base-apt/base-apt.bb
> index c9a7dad..2dba779 100644
> --- a/meta/recipes-devtools/base-apt/base-apt.bb
> +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> @@ -9,26 +9,77 @@ inherit repository
> BASE_REPO_KEY ?= ""
> KEYFILES ?= ""
>
> -do_cache_config[stamp-extra-info] = "${DISTRO}"
> -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> +populate_base_apt() {
> + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
> + # NOTE: due to packages stored by reprepro are not modified, we
> can
> + # use search by filename to check if package is already in repo.
> In
> + # addition, md5sums are compared to ensure that the package is
> the
> + # same and should not be overwritten. This method is easier and
> more
> + # robust than querying reprepro by name.
>
> -# Generate reprepro config for current distro if it doesn't exist. Once
> it's
> -# generated, this task should do nothing.
> -repo_config() {
> + # Check if this package is taken from Isar-apt, if so - ingore
> it.
> + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}"
> && \
> + continue
> +
> + # Check if this package is already in base-apt
> + ret=0
> + repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> "${package}" ||
> + ret=$?
> + [ "${ret}" = "0" ] && continue
> + if [ "${ret}" = "1" ]; then
> + repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> + "${BASE_DISTRO_CODENAME}" \
> + "${base_apt_p}"
> + fi
> +
> + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> + "${BASE_DISTRO_CODENAME}" \
> + "${package}"
> + done
> +
> + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read package;
> do
> + repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> + "${BASE_DISTRO_CODENAME}" \
> + "${package}"
> + done
> +}
> +
> +do_cache[stamp-extra-info] = "${DISTRO}"
> +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> +
> +repo() {
> repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> "${BASE_DISTRO_CODENAME}" \
> "${KEYFILES}"
> +
> + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> + # We would need two separate repository paths for that.
> + # Otherwise packages (especially the 'all' arch ones) from one
> + # distribution can influence the package versions of the other
> + # distribution.
> + bbfatal "Different host and target distributions are currently
> not supported." \
> + "Try it without cross-build."
> + fi
> +
> + populate_base_apt
> }
>
> -python do_cache_config() {
> +python do_cache() {
> + if not bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
> + return 0
> +
> for key in d.getVar('BASE_REPO_KEY').split():
> d.appendVar("SRC_URI", " %s" % key)
> fetcher = bb.fetch2.Fetch([key], d)
> filename = fetcher.localpath(key)
> d.appendVar("KEYFILES", " %s" % filename)
>
> - bb.build.exec_func('repo_config', d)
> + bb.build.exec_func('repo', d)
> }
>
> -addtask cache_config after do_unpack before do_build
> +addtask cache after do_unpack before do_build
> diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
> index 7da3ee9..733ba86 100755
> --- a/scripts/ci_build.sh
> +++ b/scripts/ci_build.sh
> @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
>
> # Enable use of signed cached base repository
> echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\" >>
> conf/local.conf
> - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
> + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
> while [ -e bitbake.sock ]; do sleep 1; done
> sudo rm -rf tmp
> sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
> sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
>
> # Enable use of unsigned cached base repository
> - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
> + bitbake $BB_ARGS $REPRO_TARGETS_SET
> while [ -e bitbake.sock ]; do sleep 1; done
> sudo rm -rf tmp
> sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> --
> 2.24.1
>
>
[-- Attachment #1.2: Type: text/html, Size: 19177 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain
2020-01-21 8:55 ` vijai kumar
@ 2020-01-21 9:54 ` Henning Schild
2020-01-21 13:01 ` vijai kumar
0 siblings, 1 reply; 68+ messages in thread
From: Henning Schild @ 2020-01-21 9:54 UTC (permalink / raw)
To: vijai kumar; +Cc: isar-users
Hi Vijai,
On Tue, 21 Jan 2020 00:55:43 -0800
vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
> Hi Henning,
>
> Can't we have the base-apt repo generation towards the end of the
> build chain?
Could you go into detail why you think the point in time matters? It
was at the end because the old implementation relied on the rootfs
hopefully keeping all .debs in its cache. While that worked it was
never really robust.
And it was optional, now it just "comes for free along the way".
Whether you use it or not is still optional.
> Right now if you need to generate a base-apt then you need to build
> twice. Once without
> ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO set.
Still the case. You will need a second run where you tell Isar to use
that repo. Only difference its creation during the first run was
implicit instead of explicit.
> What if we could just generate the repo towards the end nevertheless
> and let user decide
> whether to use the repo in the subsequent build by setting
> ISAR_USE_CACHED_BASE_REPO?
Again, i do not see where the "point in time of the first build" makes
a difference. And the user still has to decide using that interface you
mention.
Henning
> Thanks,
> Vijai Kumar K
>
>
> On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning Schild
> wrote:
> >
> > From: Henning Schild <hennin...@siemens.com <javascript:>>
> >
> > Users had to call a special task before switching to using
> > base-apt. With this commit you flip one switch and the repo will be
> > created as one of the first steps.
> >
> > The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
> > "processed" files and not just a download-cache.
> >
> > Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> > ---
> > doc/user_manual.md | 8 +--
> > meta/classes/image-cache-extension.bbclass | 64
> > ------------------ meta/classes/image.bbclass |
> > 1 - meta/classes/rootfs.bbclass | 2 +-
> > meta/conf/bitbake.conf | 4 +-
> > .../isar-bootstrap/isar-bootstrap.inc | 1 +
> > meta/recipes-devtools/base-apt/base-apt.bb | 67
> > ++++++++++++++++--- scripts/ci_build.sh |
> > 4 +- 8 files changed, 69 insertions(+), 82 deletions(-)
> > delete mode 100644 meta/classes/image-cache-extension.bbclass
> >
> > diff --git a/doc/user_manual.md b/doc/user_manual.md
> > index 4d699bd..aaf9638 100644
> > --- a/doc/user_manual.md
> > +++ b/doc/user_manual.md
> > @@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`,
> > e.g.: BASE_REPO_KEY =
> > "file://<absolute_path_to_your_pub_key_file>"' ```
> >
> > - - Trigger creation of local apt caching Debian packages during
> > image generation.
> > + - Trigger the download and caching of all required files by doing
> > a warm-up build.
> >
> > ```
> > -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
> > +bitbake mc:qemuarm-buster:isar-image-base
> > ```
> >
> > - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
> > @@ -835,14 +835,14 @@ bitbake -c cache_base_repo
> > mc:qemuarm-buster:isar-image-base
> > #ISAR_USE_CACHED_BASE_REPO ?= "1"
> > #BB_NO_NETWORK ?= "1"
> > ```
> > - - Remove build artifacts to use only local base-apt:
> > + - Remove build artifacts to use only local base-apt, in fact
> > toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as
> > well. This is just the way to be extra sure that only the download
> > cache is used.
> > ```
> > sudo rm -rf tmp
> >
> > ```
> >
> > - - Trigger again generation of image (now using local caching
> > repo):
> > + - Trigger the generation of your image again (now a local repo
> > will be created out of the download cache from the last run):
> >
> > ```
> > bitbake mc:qemuarm-buster:isar-image-base
> > diff --git a/meta/classes/image-cache-extension.bbclass
> > b/meta/classes/image-cache-extension.bbclass
> > deleted file mode 100644
> > index 4123326..0000000
> > --- a/meta/classes/image-cache-extension.bbclass
> > +++ /dev/null
> > @@ -1,64 +0,0 @@
> > -# This software is a part of ISAR.
> > -# Copyright (C) Siemens AG, 2019
> > -#
> > -# SPDX-License-Identifier: MIT
> > -#
> > -# This class extends the image.bbclass to supply the creation of
> > cache repositories
> > -
> > -inherit repository
> > -
> > -populate_base_apt() {
> > - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > package; do
> > - # NOTE: due to packages stored by reprepro are not
> > modified, we can
> > - # use search by filename to check if package is already in
> > repo. In
> > - # addition, md5sums are compared to ensure that the
> > package is the
> > - # same and should not be overwritten. This method is
> > easier and more
> > - # robust than querying reprepro by name.
> > -
> > - # Check if this package is taken from Isar-apt, if so -
> > ingore it.
> > - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > "${package}" && \
> > - continue
> > -
> > - # Check if this package is already in base-apt
> > - ret=0
> > - repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> > "${package}" ||
> > - ret=$?
> > - [ "${ret}" = "0" ] && continue
> > - if [ "${ret}" = "1" ]; then
> > - repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > - "${BASE_DISTRO_CODENAME}" \
> > - "${base_apt_p}"
> > - fi
> > -
> > - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > - "${BASE_DISTRO_CODENAME}" \
> > - "${package}"
> > - done
> > -
> > - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read
> > package; do
> > - repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > - "${BASE_DISTRO_CODENAME}" \
> > - "${package}"
> > - done
> > -}
> > -
> > -do_cache_base_repo[depends] = "base-apt:do_cache_config"
> > -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
> > -do_cache_base_repo() {
> > - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > - # We would need two separate repository paths for that.
> > - # Otherwise packages (especially the 'all' arch ones) from
> > one
> > - # distribution can influence the package versions of the
> > other
> > - # distribution.
> > - bbfatal "Different host and target distributions are
> > currently not supported." \
> > - "Try it without cross-build."
> > - fi
> > -
> > - populate_base_apt
> > -}
> > -addtask cache_base_repo after do_rootfs do_install_imager_deps
> > diff --git a/meta/classes/image.bbclass
> > b/meta/classes/image.bbclass index b687fcf..cfd617a 100644
> > --- a/meta/classes/image.bbclass
> > +++ b/meta/classes/image.bbclass
> > @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?=
> > "${DEPLOY_DIR_IMAGE}"
> > inherit rootfs
> > inherit image-sdk-extension
> > -inherit image-cache-extension
> > inherit image-tools-extension
> > inherit image-postproc-extension
> > inherit image-locales-extension
> > diff --git a/meta/classes/rootfs.bbclass
> > b/meta/classes/rootfs.bbclass index 6d4d42c..a10e7cc 100644
> > --- a/meta/classes/rootfs.bbclass
> > +++ b/meta/classes/rootfs.bbclass
> > @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
> >
> > do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
> > do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
> > ${ROOTFS_INSTALL_COMMAND}"
> > -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > 'host'}:do_build isar-apt:do_cache_config"
> > +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
> > do_rootfs_install[deptask] = "do_deploy_deb"
> > python do_rootfs_install() {
> > configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True)
> > or "").split()
> > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > index d86c5b9..e46142d 100644
> > --- a/meta/conf/bitbake.conf
> > +++ b/meta/conf/bitbake.conf
> > @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
> > THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
> >
> > # Base apt repository paths
> > -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
> > -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
> > +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
> > +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
> >
> > BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH
> > BBSERVER DL_DIR \
> > SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME
> > SHELL TERM \
> > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > index 5b14cd3..57cdea9 100644
> > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > @@ -219,6 +219,7 @@ def get_host_release():
> >
> > do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> > do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> > +do_bootstrap[depends] = "base-apt:do_cache"
> >
> > isar_bootstrap() {
> > deb_dl_dir_import "${ROOTFSDIR}"
> > diff --git a/meta/recipes-devtools/base-apt/base-apt.bb
> > b/meta/recipes-devtools/base-apt/base-apt.bb
> > index c9a7dad..2dba779 100644
> > --- a/meta/recipes-devtools/base-apt/base-apt.bb
> > +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> > @@ -9,26 +9,77 @@ inherit repository
> > BASE_REPO_KEY ?= ""
> > KEYFILES ?= ""
> >
> > -do_cache_config[stamp-extra-info] = "${DISTRO}"
> > -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > +populate_base_apt() {
> > + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > package; do
> > + # NOTE: due to packages stored by reprepro are not
> > modified, we can
> > + # use search by filename to check if package is already in
> > repo. In
> > + # addition, md5sums are compared to ensure that the
> > package is the
> > + # same and should not be overwritten. This method is
> > easier and more
> > + # robust than querying reprepro by name.
> >
> > -# Generate reprepro config for current distro if it doesn't exist.
> > Once it's
> > -# generated, this task should do nothing.
> > -repo_config() {
> > + # Check if this package is taken from Isar-apt, if so -
> > ingore it.
> > + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > "${package}" && \
> > + continue
> > +
> > + # Check if this package is already in base-apt
> > + ret=0
> > + repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> > "${package}" ||
> > + ret=$?
> > + [ "${ret}" = "0" ] && continue
> > + if [ "${ret}" = "1" ]; then
> > + repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > + "${BASE_DISTRO_CODENAME}" \
> > + "${base_apt_p}"
> > + fi
> > +
> > + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > + "${BASE_DISTRO_CODENAME}" \
> > + "${package}"
> > + done
> > +
> > + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read
> > package; do
> > + repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > + "${BASE_DISTRO_CODENAME}" \
> > + "${package}"
> > + done
> > +}
> > +
> > +do_cache[stamp-extra-info] = "${DISTRO}"
> > +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > +
> > +repo() {
> > repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > "${BASE_DISTRO_CODENAME}" \
> > "${KEYFILES}"
> > +
> > + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > + # We would need two separate repository paths for that.
> > + # Otherwise packages (especially the 'all' arch ones) from
> > one
> > + # distribution can influence the package versions of the
> > other
> > + # distribution.
> > + bbfatal "Different host and target distributions are
> > currently not supported." \
> > + "Try it without cross-build."
> > + fi
> > +
> > + populate_base_apt
> > }
> >
> > -python do_cache_config() {
> > +python do_cache() {
> > + if not
> > bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
> > + return 0
> > +
> > for key in d.getVar('BASE_REPO_KEY').split():
> > d.appendVar("SRC_URI", " %s" % key)
> > fetcher = bb.fetch2.Fetch([key], d)
> > filename = fetcher.localpath(key)
> > d.appendVar("KEYFILES", " %s" % filename)
> >
> > - bb.build.exec_func('repo_config', d)
> > + bb.build.exec_func('repo', d)
> > }
> >
> > -addtask cache_config after do_unpack before do_build
> > +addtask cache after do_unpack before do_build
> > diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
> > index 7da3ee9..733ba86 100755
> > --- a/scripts/ci_build.sh
> > +++ b/scripts/ci_build.sh
> > @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
> >
> > # Enable use of signed cached base repository
> > echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\"
> > >> conf/local.conf
> > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
> > + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
> > while [ -e bitbake.sock ]; do sleep 1; done
> > sudo rm -rf tmp
> > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
> > sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
> >
> > # Enable use of unsigned cached base repository
> > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
> > + bitbake $BB_ARGS $REPRO_TARGETS_SET
> > while [ -e bitbake.sock ]; do sleep 1; done
> > sudo rm -rf tmp
> > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > --
> > 2.24.1
> >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain
2020-01-21 9:54 ` Henning Schild
@ 2020-01-21 13:01 ` vijai kumar
2020-01-27 7:32 ` Henning Schild
0 siblings, 1 reply; 68+ messages in thread
From: vijai kumar @ 2020-01-21 13:01 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 16614 bytes --]
Hi Henning,
On Tuesday, January 21, 2020 at 3:24:20 PM UTC+5:30, Henning Schild wrote:
>
> Hi Vijai,
>
> On Tue, 21 Jan 2020 00:55:43 -0800
> vijai kumar <vijaikumar...@gmail.com <javascript:>> wrote:
>
> > Hi Henning,
> >
> > Can't we have the base-apt repo generation towards the end of the
> > build chain?
>
> Could you go into detail why you think the point in time matters? It
> was at the end because the old implementation relied on the rootfs
> hopefully keeping all .debs in its cache. While that worked it was
> never really robust.
>
> And it was optional, now it just "comes for free along the way".
> Whether you use it or not is still optional.
>
> > Right now if you need to generate a base-apt then you need to build
> > twice. Once without
> > ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO set.
>
> Still the case. You will need a second run where you tell Isar to use
> that repo. Only difference its creation during the first run was
> implicit instead of explicit.
>
> > What if we could just generate the repo towards the end nevertheless
> > and let user decide
> > whether to use the repo in the subsequent build by setting
> > ISAR_USE_CACHED_BASE_REPO?
>
> Again, i do not see where the "point in time of the first build" makes
> a difference. And the user still has to decide using that interface you
> mention.
>
>
We are also generating base-apt for redistribution to customers. So far
we have relied on the single-step process of using cache_base_repo.
Now this has become 2 steps for us.
Run builds for all the boards and a dummy build to populate the base-apt(?).
Which very well could be avoided if the base-apt is updated incrementally
during subsequent builds.
I was more curious about the extra step needed for this use-case.
I don't have a solid reason for why it should be done at the end at this
point
in time.
Vijai Kumar K
Henning
>
> > Thanks,
> > Vijai Kumar K
> >
> >
> > On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning Schild
> > wrote:
> > >
> > > From: Henning Schild <hennin...@siemens.com <javascript:>>
> > >
> > > Users had to call a special task before switching to using
> > > base-apt. With this commit you flip one switch and the repo will be
> > > created as one of the first steps.
> > >
> > > The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
> > > "processed" files and not just a download-cache.
> > >
> > > Signed-off-by: Henning Schild <hennin...@siemens.com <javascript:>>
> > > ---
> > > doc/user_manual.md | 8 +--
> > > meta/classes/image-cache-extension.bbclass | 64
> > > ------------------ meta/classes/image.bbclass |
> > > 1 - meta/classes/rootfs.bbclass | 2 +-
> > > meta/conf/bitbake.conf | 4 +-
> > > .../isar-bootstrap/isar-bootstrap.inc | 1 +
> > > meta/recipes-devtools/base-apt/base-apt.bb | 67
> > > ++++++++++++++++--- scripts/ci_build.sh |
> > > 4 +- 8 files changed, 69 insertions(+), 82 deletions(-)
> > > delete mode 100644 meta/classes/image-cache-extension.bbclass
> > >
> > > diff --git a/doc/user_manual.md b/doc/user_manual.md
> > > index 4d699bd..aaf9638 100644
> > > --- a/doc/user_manual.md
> > > +++ b/doc/user_manual.md
> > > @@ -822,10 +822,10 @@ path to the public key in `conf/local.conf`,
> > > e.g.: BASE_REPO_KEY =
> > > "file://<absolute_path_to_your_pub_key_file>"' ```
> > >
> > > - - Trigger creation of local apt caching Debian packages during
> > > image generation.
> > > + - Trigger the download and caching of all required files by doing
> > > a warm-up build.
> > >
> > > ```
> > > -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
> > > +bitbake mc:qemuarm-buster:isar-image-base
> > > ```
> > >
> > > - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
> > > @@ -835,14 +835,14 @@ bitbake -c cache_base_repo
> > > mc:qemuarm-buster:isar-image-base
> > > #ISAR_USE_CACHED_BASE_REPO ?= "1"
> > > #BB_NO_NETWORK ?= "1"
> > > ```
> > > - - Remove build artifacts to use only local base-apt:
> > > + - Remove build artifacts to use only local base-apt, in fact
> > > toggling ISAR_USE_CACHED_BASE_REPO should trigger a full rebuild as
> > > well. This is just the way to be extra sure that only the download
> > > cache is used.
> > > ```
> > > sudo rm -rf tmp
> > >
> > > ```
> > >
> > > - - Trigger again generation of image (now using local caching
> > > repo):
> > > + - Trigger the generation of your image again (now a local repo
> > > will be created out of the download cache from the last run):
> > >
> > > ```
> > > bitbake mc:qemuarm-buster:isar-image-base
> > > diff --git a/meta/classes/image-cache-extension.bbclass
> > > b/meta/classes/image-cache-extension.bbclass
> > > deleted file mode 100644
> > > index 4123326..0000000
> > > --- a/meta/classes/image-cache-extension.bbclass
> > > +++ /dev/null
> > > @@ -1,64 +0,0 @@
> > > -# This software is a part of ISAR.
> > > -# Copyright (C) Siemens AG, 2019
> > > -#
> > > -# SPDX-License-Identifier: MIT
> > > -#
> > > -# This class extends the image.bbclass to supply the creation of
> > > cache repositories
> > > -
> > > -inherit repository
> > > -
> > > -populate_base_apt() {
> > > - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > > package; do
> > > - # NOTE: due to packages stored by reprepro are not
> > > modified, we can
> > > - # use search by filename to check if package is already in
> > > repo. In
> > > - # addition, md5sums are compared to ensure that the
> > > package is the
> > > - # same and should not be overwritten. This method is
> > > easier and more
> > > - # robust than querying reprepro by name.
> > > -
> > > - # Check if this package is taken from Isar-apt, if so -
> > > ingore it.
> > > - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > > "${package}" && \
> > > - continue
> > > -
> > > - # Check if this package is already in base-apt
> > > - ret=0
> > > - repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> > > "${package}" ||
> > > - ret=$?
> > > - [ "${ret}" = "0" ] && continue
> > > - if [ "${ret}" = "1" ]; then
> > > - repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > - "${BASE_DISTRO_CODENAME}" \
> > > - "${base_apt_p}"
> > > - fi
> > > -
> > > - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > - "${BASE_DISTRO_CODENAME}" \
> > > - "${package}"
> > > - done
> > > -
> > > - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read
> > > package; do
> > > - repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > - "${BASE_DISTRO_CODENAME}" \
> > > - "${package}"
> > > - done
> > > -}
> > > -
> > > -do_cache_base_repo[depends] = "base-apt:do_cache_config"
> > > -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
> > > -do_cache_base_repo() {
> > > - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > > - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > > - # We would need two separate repository paths for that.
> > > - # Otherwise packages (especially the 'all' arch ones) from
> > > one
> > > - # distribution can influence the package versions of the
> > > other
> > > - # distribution.
> > > - bbfatal "Different host and target distributions are
> > > currently not supported." \
> > > - "Try it without cross-build."
> > > - fi
> > > -
> > > - populate_base_apt
> > > -}
> > > -addtask cache_base_repo after do_rootfs do_install_imager_deps
> > > diff --git a/meta/classes/image.bbclass
> > > b/meta/classes/image.bbclass index b687fcf..cfd617a 100644
> > > --- a/meta/classes/image.bbclass
> > > +++ b/meta/classes/image.bbclass
> > > @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?=
> > > "${DEPLOY_DIR_IMAGE}"
> > > inherit rootfs
> > > inherit image-sdk-extension
> > > -inherit image-cache-extension
> > > inherit image-tools-extension
> > > inherit image-postproc-extension
> > > inherit image-locales-extension
> > > diff --git a/meta/classes/rootfs.bbclass
> > > b/meta/classes/rootfs.bbclass index 6d4d42c..a10e7cc 100644
> > > --- a/meta/classes/rootfs.bbclass
> > > +++ b/meta/classes/rootfs.bbclass
> > > @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
> > >
> > > do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
> > > do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
> > > ${ROOTFS_INSTALL_COMMAND}"
> > > -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > > 'host'}:do_build isar-apt:do_cache_config"
> > > +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > > 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
> > > do_rootfs_install[deptask] = "do_deploy_deb"
> > > python do_rootfs_install() {
> > > configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND", True)
> > > or "").split()
> > > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > > index d86c5b9..e46142d 100644
> > > --- a/meta/conf/bitbake.conf
> > > +++ b/meta/conf/bitbake.conf
> > > @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR = "${DEPLOY_DIR}/isar-apt/db"
> > > THIRD_PARTY_APT_KEYRING = "/etc/apt/trusted.gpg.d/third_party.gpg"
> > >
> > > # Base apt repository paths
> > > -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
> > > -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
> > > +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
> > > +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
> > >
> > > BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH
> > > BBSERVER DL_DIR \
> > > SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME
> > > SHELL TERM \
> > > diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > > index 5b14cd3..57cdea9 100644
> > > --- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > > +++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > > @@ -219,6 +219,7 @@ def get_host_release():
> > >
> > > do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> > > do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> > > +do_bootstrap[depends] = "base-apt:do_cache"
> > >
> > > isar_bootstrap() {
> > > deb_dl_dir_import "${ROOTFSDIR}"
> > > diff --git a/meta/recipes-devtools/base-apt/base-apt.bb
> > > b/meta/recipes-devtools/base-apt/base-apt.bb
> > > index c9a7dad..2dba779 100644
> > > --- a/meta/recipes-devtools/base-apt/base-apt.bb
> > > +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> > > @@ -9,26 +9,77 @@ inherit repository
> > > BASE_REPO_KEY ?= ""
> > > KEYFILES ?= ""
> > >
> > > -do_cache_config[stamp-extra-info] = "${DISTRO}"
> > > -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > +populate_base_apt() {
> > > + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > > package; do
> > > + # NOTE: due to packages stored by reprepro are not
> > > modified, we can
> > > + # use search by filename to check if package is already in
> > > repo. In
> > > + # addition, md5sums are compared to ensure that the
> > > package is the
> > > + # same and should not be overwritten. This method is
> > > easier and more
> > > + # robust than querying reprepro by name.
> > >
> > > -# Generate reprepro config for current distro if it doesn't exist.
> > > Once it's
> > > -# generated, this task should do nothing.
> > > -repo_config() {
> > > + # Check if this package is taken from Isar-apt, if so -
> > > ingore it.
> > > + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > > "${package}" && \
> > > + continue
> > > +
> > > + # Check if this package is already in base-apt
> > > + ret=0
> > > + repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}"
> > > "${package}" ||
> > > + ret=$?
> > > + [ "${ret}" = "0" ] && continue
> > > + if [ "${ret}" = "1" ]; then
> > > + repo_del_package "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > + "${BASE_DISTRO_CODENAME}" \
> > > + "${base_apt_p}"
> > > + fi
> > > +
> > > + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > + "${BASE_DISTRO_CODENAME}" \
> > > + "${package}"
> > > + done
> > > +
> > > + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while read
> > > package; do
> > > + repo_add_srcpackage "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > + "${BASE_DISTRO_CODENAME}" \
> > > + "${package}"
> > > + done
> > > +}
> > > +
> > > +do_cache[stamp-extra-info] = "${DISTRO}"
> > > +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > +
> > > +repo() {
> > > repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > "${BASE_DISTRO_CODENAME}" \
> > > "${KEYFILES}"
> > > +
> > > + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > > + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > > + # We would need two separate repository paths for that.
> > > + # Otherwise packages (especially the 'all' arch ones) from
> > > one
> > > + # distribution can influence the package versions of the
> > > other
> > > + # distribution.
> > > + bbfatal "Different host and target distributions are
> > > currently not supported." \
> > > + "Try it without cross-build."
> > > + fi
> > > +
> > > + populate_base_apt
> > > }
> > >
> > > -python do_cache_config() {
> > > +python do_cache() {
> > > + if not
> > > bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
> > > + return 0
> > > +
> > > for key in d.getVar('BASE_REPO_KEY').split():
> > > d.appendVar("SRC_URI", " %s" % key)
> > > fetcher = bb.fetch2.Fetch([key], d)
> > > filename = fetcher.localpath(key)
> > > d.appendVar("KEYFILES", " %s" % filename)
> > >
> > > - bb.build.exec_func('repo_config', d)
> > > + bb.build.exec_func('repo', d)
> > > }
> > >
> > > -addtask cache_config after do_unpack before do_build
> > > +addtask cache after do_unpack before do_build
> > > diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
> > > index 7da3ee9..733ba86 100755
> > > --- a/scripts/ci_build.sh
> > > +++ b/scripts/ci_build.sh
> > > @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
> > >
> > > # Enable use of signed cached base repository
> > > echo BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\"
> > > >> conf/local.conf
> > > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET_SIGNED
> > > + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
> > > while [ -e bitbake.sock ]; do sleep 1; done
> > > sudo rm -rf tmp
> > > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > > @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
> > > sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g' conf/local.conf
> > >
> > > # Enable use of unsigned cached base repository
> > > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
> > > + bitbake $BB_ARGS $REPRO_TARGETS_SET
> > > while [ -e bitbake.sock ]; do sleep 1; done
> > > sudo rm -rf tmp
> > > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > > --
> > > 2.24.1
> > >
> > >
> >
>
>
[-- Attachment #1.2: Type: text/html, Size: 24835 bytes --]
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain
2020-01-21 13:01 ` vijai kumar
@ 2020-01-27 7:32 ` Henning Schild
0 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2020-01-27 7:32 UTC (permalink / raw)
To: vijai kumar; +Cc: isar-users
On Tue, 21 Jan 2020 05:01:32 -0800
vijai kumar <vijaikumar.kanagarajan@gmail.com> wrote:
> Hi Henning,
>
> On Tuesday, January 21, 2020 at 3:24:20 PM UTC+5:30, Henning Schild
> wrote:
> >
> > Hi Vijai,
> >
> > On Tue, 21 Jan 2020 00:55:43 -0800
> > vijai kumar <vijaikumar...@gmail.com <javascript:>> wrote:
> >
> > > Hi Henning,
> > >
> > > Can't we have the base-apt repo generation towards the end of the
> > > build chain?
> >
> > Could you go into detail why you think the point in time matters?
> > It was at the end because the old implementation relied on the
> > rootfs hopefully keeping all .debs in its cache. While that worked
> > it was never really robust.
> >
> > And it was optional, now it just "comes for free along the way".
> > Whether you use it or not is still optional.
> >
> > > Right now if you need to generate a base-apt then you need to
> > > build twice. Once without
> > > ISAR_USE_CACHED_BASE_REPO and once with ISAR_USE_CACHED_BASE_REPO
> > > set.
> >
> > Still the case. You will need a second run where you tell Isar to
> > use that repo. Only difference its creation during the first run
> > was implicit instead of explicit.
> >
> > > What if we could just generate the repo towards the end
> > > nevertheless and let user decide
> > > whether to use the repo in the subsequent build by setting
> > > ISAR_USE_CACHED_BASE_REPO?
> >
> > Again, i do not see where the "point in time of the first build"
> > makes a difference. And the user still has to decide using that
> > interface you mention.
> >
> >
> We are also generating base-apt for redistribution to customers. So
> far we have relied on the single-step process of using
> cache_base_repo.
That is one - if not the main - intention behind it.
> Now this has become 2 steps for us.
Got it. But in fact it should always have been two steps to actually
"validate" it is somehow working as expected.
In the second build "bitbake base-apt -c do_cache" should quickly get
you to the unverified state you had before.
> Run builds for all the boards and a dummy build to populate the
> base-apt(?). Which very well could be avoided if the base-apt is
> updated incrementally during subsequent builds.
Note that base-apt is and always has been a mix of all packages used.
So it does include build dependencies (which might have to get cleared
before redistribution), and it contains all packages from all images
built in one "tree". My conservative suggestion would be to not build
multiple images in one tree when generating base-apt. And after these
patches you probably want to start with a fresh DL_DIR to not include
packages from previous builds.
https://groups.google.com/d/msg/isar-users/8djs0Thl0wI/n5Bo2hPeCgAJ
Henning
> I was more curious about the extra step needed for this use-case.
> I don't have a solid reason for why it should be done at the end at
> this point
> in time.
>
> Vijai Kumar K
>
> Henning
> >
> > > Thanks,
> > > Vijai Kumar K
> > >
> > >
> > > On Monday, December 16, 2019 at 5:20:16 PM UTC+5:30, Henning
> > > Schild wrote:
> > > >
> > > > From: Henning Schild <hennin...@siemens.com <javascript:>>
> > > >
> > > > Users had to call a special task before switching to using
> > > > base-apt. With this commit you flip one switch and the repo
> > > > will be created as one of the first steps.
> > > >
> > > > The base-apt repo gets moved from DL_DIR to DEPLOY_DIR. It is
> > > > "processed" files and not just a download-cache.
> > > >
> > > > Signed-off-by: Henning Schild <hennin...@siemens.com
> > > > <javascript:>> ---
> > > > doc/user_manual.md | 8 +--
> > > > meta/classes/image-cache-extension.bbclass | 64
> > > > ------------------ meta/classes/image.bbclass
> > > > | 1 - meta/classes/rootfs.bbclass | 2 +-
> > > > meta/conf/bitbake.conf | 4 +-
> > > > .../isar-bootstrap/isar-bootstrap.inc | 1 +
> > > > meta/recipes-devtools/base-apt/base-apt.bb | 67
> > > > ++++++++++++++++--- scripts/ci_build.sh
> > > > | 4 +- 8 files changed, 69 insertions(+), 82 deletions(-)
> > > > delete mode 100644 meta/classes/image-cache-extension.bbclass
> > > >
> > > > diff --git a/doc/user_manual.md b/doc/user_manual.md
> > > > index 4d699bd..aaf9638 100644
> > > > --- a/doc/user_manual.md
> > > > +++ b/doc/user_manual.md
> > > > @@ -822,10 +822,10 @@ path to the public key in
> > > > `conf/local.conf`, e.g.: BASE_REPO_KEY =
> > > > "file://<absolute_path_to_your_pub_key_file>"' ```
> > > >
> > > > - - Trigger creation of local apt caching Debian packages
> > > > during image generation.
> > > > + - Trigger the download and caching of all required files by
> > > > doing a warm-up build.
> > > >
> > > > ```
> > > > -bitbake -c cache_base_repo mc:qemuarm-buster:isar-image-base
> > > > +bitbake mc:qemuarm-buster:isar-image-base
> > > > ```
> > > >
> > > > - Set `ISAR_USE_CACHED_BASE_REPO` in `conf/local.conf`:
> > > > @@ -835,14 +835,14 @@ bitbake -c cache_base_repo
> > > > mc:qemuarm-buster:isar-image-base
> > > > #ISAR_USE_CACHED_BASE_REPO ?= "1"
> > > > #BB_NO_NETWORK ?= "1"
> > > > ```
> > > > - - Remove build artifacts to use only local base-apt:
> > > > + - Remove build artifacts to use only local base-apt, in fact
> > > > toggling ISAR_USE_CACHED_BASE_REPO should trigger a full
> > > > rebuild as well. This is just the way to be extra sure that
> > > > only the download cache is used.
> > > > ```
> > > > sudo rm -rf tmp
> > > >
> > > > ```
> > > >
> > > > - - Trigger again generation of image (now using local caching
> > > > repo):
> > > > + - Trigger the generation of your image again (now a local
> > > > repo will be created out of the download cache from the last
> > > > run):
> > > > ```
> > > > bitbake mc:qemuarm-buster:isar-image-base
> > > > diff --git a/meta/classes/image-cache-extension.bbclass
> > > > b/meta/classes/image-cache-extension.bbclass
> > > > deleted file mode 100644
> > > > index 4123326..0000000
> > > > --- a/meta/classes/image-cache-extension.bbclass
> > > > +++ /dev/null
> > > > @@ -1,64 +0,0 @@
> > > > -# This software is a part of ISAR.
> > > > -# Copyright (C) Siemens AG, 2019
> > > > -#
> > > > -# SPDX-License-Identifier: MIT
> > > > -#
> > > > -# This class extends the image.bbclass to supply the creation
> > > > of cache repositories
> > > > -
> > > > -inherit repository
> > > > -
> > > > -populate_base_apt() {
> > > > - find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > > > package; do
> > > > - # NOTE: due to packages stored by reprepro are not
> > > > modified, we can
> > > > - # use search by filename to check if package is
> > > > already in repo. In
> > > > - # addition, md5sums are compared to ensure that the
> > > > package is the
> > > > - # same and should not be overwritten. This method is
> > > > easier and more
> > > > - # robust than querying reprepro by name.
> > > > -
> > > > - # Check if this package is taken from Isar-apt, if so
> > > > - ingore it.
> > > > - repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > > > "${package}" && \
> > > > - continue
> > > > -
> > > > - # Check if this package is already in base-apt
> > > > - ret=0
> > > > - repo_contains_package
> > > > "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
> > > > - ret=$?
> > > > - [ "${ret}" = "0" ] && continue
> > > > - if [ "${ret}" = "1" ]; then
> > > > - repo_del_package
> > > > "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > - "${BASE_DISTRO_CODENAME}" \
> > > > - "${base_apt_p}"
> > > > - fi
> > > > -
> > > > - repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}"
> > > > \
> > > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > - "${BASE_DISTRO_CODENAME}" \
> > > > - "${package}"
> > > > - done
> > > > -
> > > > - find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while
> > > > read package; do
> > > > - repo_add_srcpackage
> > > > "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > > - "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > - "${BASE_DISTRO_CODENAME}" \
> > > > - "${package}"
> > > > - done
> > > > -}
> > > > -
> > > > -do_cache_base_repo[depends] = "base-apt:do_cache_config"
> > > > -do_cache_base_repo[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > > -do_cache_base_repo[stamp-extra-info] = "${DISTRO}-${MACHINE}"
> > > > -do_cache_base_repo() {
> > > > - if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > > > - [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > > > - # We would need two separate repository paths for
> > > > that.
> > > > - # Otherwise packages (especially the 'all' arch ones)
> > > > from one
> > > > - # distribution can influence the package versions of
> > > > the other
> > > > - # distribution.
> > > > - bbfatal "Different host and target distributions are
> > > > currently not supported." \
> > > > - "Try it without cross-build."
> > > > - fi
> > > > -
> > > > - populate_base_apt
> > > > -}
> > > > -addtask cache_base_repo after do_rootfs do_install_imager_deps
> > > > diff --git a/meta/classes/image.bbclass
> > > > b/meta/classes/image.bbclass index b687fcf..cfd617a 100644
> > > > --- a/meta/classes/image.bbclass
> > > > +++ b/meta/classes/image.bbclass
> > > > @@ -66,7 +66,6 @@ ROOTFS_MANIFEST_DEPLOY_DIR ?=
> > > > "${DEPLOY_DIR_IMAGE}"
> > > > inherit rootfs
> > > > inherit image-sdk-extension
> > > > -inherit image-cache-extension
> > > > inherit image-tools-extension
> > > > inherit image-postproc-extension
> > > > inherit image-locales-extension
> > > > diff --git a/meta/classes/rootfs.bbclass
> > > > b/meta/classes/rootfs.bbclass index 6d4d42c..a10e7cc 100644
> > > > --- a/meta/classes/rootfs.bbclass
> > > > +++ b/meta/classes/rootfs.bbclass
> > > > @@ -151,7 +151,7 @@ rootfs_install_pkgs_install() {
> > > >
> > > > do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
> > > > do_rootfs_install[vardeps] = "${ROOTFS_CONFIGURE_COMMAND}
> > > > ${ROOTFS_INSTALL_COMMAND}"
> > > > -do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > > > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > > > 'host'}:do_build isar-apt:do_cache_config"
> > > > +do_rootfs_install[depends] = "isar-bootstrap-${@'target' if
> > > > d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else
> > > > 'host'}:do_build isar-apt:do_cache_config base-apt:do_cache"
> > > > do_rootfs_install[deptask] = "do_deploy_deb"
> > > > python do_rootfs_install() {
> > > > configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND",
> > > > True) or "").split()
> > > > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> > > > index d86c5b9..e46142d 100644
> > > > --- a/meta/conf/bitbake.conf
> > > > +++ b/meta/conf/bitbake.conf
> > > > @@ -80,8 +80,8 @@ REPO_ISAR_DB_DIR =
> > > > "${DEPLOY_DIR}/isar-apt/db" THIRD_PARTY_APT_KEYRING =
> > > > "/etc/apt/trusted.gpg.d/third_party.gpg"
> > > > # Base apt repository paths
> > > > -REPO_BASE_DIR = "${DL_DIR}/base-apt/${DISTRO}/apt"
> > > > -REPO_BASE_DB_DIR = "${DL_DIR}/base-apt/${DISTRO}/db"
> > > > +REPO_BASE_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/apt"
> > > > +REPO_BASE_DB_DIR = "${DEPLOY_DIR}/base-apt/${DISTRO}/db"
> > > >
> > > > BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH
> > > > BBPATH BBSERVER DL_DIR \
> > > > SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME
> > > > LOGNAME SHELL TERM \
> > > > diff --git
> > > > a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
> > > > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc index
> > > > 5b14cd3..57cdea9 100644 ---
> > > > a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc +++
> > > > b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc @@ -219,6
> > > > +219,7 @@ def get_host_release():
> > > > do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
> > > > do_bootstrap[dirs] = "${DEPLOY_DIR_BOOTSTRAP}"
> > > > +do_bootstrap[depends] = "base-apt:do_cache"
> > > >
> > > > isar_bootstrap() {
> > > > deb_dl_dir_import "${ROOTFSDIR}"
> > > > diff --git a/meta/recipes-devtools/base-apt/base-apt.bb
> > > > b/meta/recipes-devtools/base-apt/base-apt.bb
> > > > index c9a7dad..2dba779 100644
> > > > --- a/meta/recipes-devtools/base-apt/base-apt.bb
> > > > +++ b/meta/recipes-devtools/base-apt/base-apt.bb
> > > > @@ -9,26 +9,77 @@ inherit repository
> > > > BASE_REPO_KEY ?= ""
> > > > KEYFILES ?= ""
> > > >
> > > > -do_cache_config[stamp-extra-info] = "${DISTRO}"
> > > > -do_cache_config[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > > +populate_base_apt() {
> > > > + find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read
> > > > package; do
> > > > + # NOTE: due to packages stored by reprepro are not
> > > > modified, we can
> > > > + # use search by filename to check if package is
> > > > already in repo. In
> > > > + # addition, md5sums are compared to ensure that the
> > > > package is the
> > > > + # same and should not be overwritten. This method is
> > > > easier and more
> > > > + # robust than querying reprepro by name.
> > > >
> > > > -# Generate reprepro config for current distro if it doesn't
> > > > exist. Once it's
> > > > -# generated, this task should do nothing.
> > > > -repo_config() {
> > > > + # Check if this package is taken from Isar-apt, if so
> > > > - ingore it.
> > > > + repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}"
> > > > "${package}" && \
> > > > + continue
> > > > +
> > > > + # Check if this package is already in base-apt
> > > > + ret=0
> > > > + repo_contains_package
> > > > "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
> > > > + ret=$?
> > > > + [ "${ret}" = "0" ] && continue
> > > > + if [ "${ret}" = "1" ]; then
> > > > + repo_del_package
> > > > "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > + "${BASE_DISTRO_CODENAME}" \
> > > > + "${base_apt_p}"
> > > > + fi
> > > > +
> > > > + repo_add_packages "${REPO_BASE_DIR}"/"${BASE_DISTRO}"
> > > > \
> > > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > + "${BASE_DISTRO_CODENAME}" \
> > > > + "${package}"
> > > > + done
> > > > +
> > > > + find "${DEBSRCDIR}"/"${DISTRO}" -name '*\.dsc' | while
> > > > read package; do
> > > > + repo_add_srcpackage
> > > > "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > > + "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > + "${BASE_DISTRO_CODENAME}" \
> > > > + "${package}"
> > > > + done
> > > > +}
> > > > +
> > > > +do_cache[stamp-extra-info] = "${DISTRO}"
> > > > +do_cache[lockfiles] = "${REPO_BASE_DIR}/isar.lock"
> > > > +
> > > > +repo() {
> > > > repo_create "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
> > > > "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}" \
> > > > "${BASE_DISTRO_CODENAME}" \
> > > > "${KEYFILES}"
> > > > +
> > > > + if [ -d '${BUILDCHROOT_HOST_DIR}/var/cache/apt' ] &&
> > > > + [ '${DISTRO}' != '${HOST_DISTRO}' ]; then
> > > > + # We would need two separate repository paths for
> > > > that.
> > > > + # Otherwise packages (especially the 'all' arch ones)
> > > > from one
> > > > + # distribution can influence the package versions of
> > > > the other
> > > > + # distribution.
> > > > + bbfatal "Different host and target distributions are
> > > > currently not supported." \
> > > > + "Try it without cross-build."
> > > > + fi
> > > > +
> > > > + populate_base_apt
> > > > }
> > > >
> > > > -python do_cache_config() {
> > > > +python do_cache() {
> > > > + if not
> > > > bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')):
> > > > + return 0
> > > > +
> > > > for key in d.getVar('BASE_REPO_KEY').split():
> > > > d.appendVar("SRC_URI", " %s" % key)
> > > > fetcher = bb.fetch2.Fetch([key], d)
> > > > filename = fetcher.localpath(key)
> > > > d.appendVar("KEYFILES", " %s" % filename)
> > > >
> > > > - bb.build.exec_func('repo_config', d)
> > > > + bb.build.exec_func('repo', d)
> > > > }
> > > >
> > > > -addtask cache_config after do_unpack before do_build
> > > > +addtask cache after do_unpack before do_build
> > > > diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
> > > > index 7da3ee9..733ba86 100755
> > > > --- a/scripts/ci_build.sh
> > > > +++ b/scripts/ci_build.sh
> > > > @@ -145,7 +145,7 @@ if [ -n "$REPRO_BUILD" ]; then
> > > >
> > > > # Enable use of signed cached base repository
> > > > echo
> > > > BASE_REPO_KEY=\"file://$ISAR_TESTSUITE_GPG_PUB_KEY_FILE\"
> > > > >> conf/local.conf
> > > > - bitbake $BB_ARGS -c cache_base_repo
> > > > $REPRO_TARGETS_SET_SIGNED
> > > > + bitbake $BB_ARGS $REPRO_TARGETS_SET_SIGNED
> > > > while [ -e bitbake.sock ]; do sleep 1; done
> > > > sudo rm -rf tmp
> > > > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > > > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > > > @@ -159,7 +159,7 @@ if [ -n "$REPRO_BUILD" ]; then
> > > > sed -i -e 's/^BASE_REPO_KEY/#BASE_REPO_KEY/g'
> > > > conf/local.conf
> > > > # Enable use of unsigned cached base repository
> > > > - bitbake $BB_ARGS -c cache_base_repo $REPRO_TARGETS_SET
> > > > + bitbake $BB_ARGS $REPRO_TARGETS_SET
> > > > while [ -e bitbake.sock ]; do sleep 1; done
> > > > sudo rm -rf tmp
> > > > sed -i -e 's/#ISAR_USE_CACHED_BASE_REPO ?=
> > > > "1"/ISAR_USE_CACHED_BASE_REPO ?= "1"/g' conf/local.conf
> > > > --
> > > > 2.24.1
> > > >
> > > >
> > >
> >
> >
>
^ permalink raw reply [flat|nested] 68+ messages in thread
* [PATCHv2 24/25] meta: base-apt: remove isar-apt check from population
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (22 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 23/25] base-apt: pull base-apt population to the front of the build chain Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-16 11:50 ` [PATCHv2 25/25] meta: deb-dl-dir: only export newly downloaded files Henning Schild
2019-12-19 9:21 ` [PATCHv2 00/25] base-apt-rework Henning Schild
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Now that we populate just from the DL_DIR and filter out the isar-apt
packages when populating this, we can skip that test since it will never
match.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/recipes-devtools/base-apt/base-apt.bb | 4 ----
1 file changed, 4 deletions(-)
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 2dba779..cfc26ed 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -17,10 +17,6 @@ populate_base_apt() {
# same and should not be overwritten. This method is easier and more
# robust than querying reprepro by name.
- # Check if this package is taken from Isar-apt, if so - ingore it.
- repo_contains_package "${REPO_ISAR_DIR}/${DISTRO}" "${package}" && \
- continue
-
# Check if this package is already in base-apt
ret=0
repo_contains_package "${REPO_BASE_DIR}/${BASE_DISTRO}" "${package}" ||
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* [PATCHv2 25/25] meta: deb-dl-dir: only export newly downloaded files
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (23 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 24/25] meta: base-apt: remove isar-apt check from population Henning Schild
@ 2019-12-16 11:50 ` Henning Schild
2019-12-19 9:21 ` [PATCHv2 00/25] base-apt-rework Henning Schild
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-16 11:50 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger, Henning Schild
From: Henning Schild <henning.schild@siemens.com>
Remember the time we imported the previously downloaded files, and only
copy out new ones when exporting again. That saved pointless copies and
time.
Signed-off-by: Henning Schild <henning.schild@siemens.com>
---
meta/classes/deb-dl-dir.bbclass | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index f969960..e65fc51 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -8,24 +8,31 @@ inherit repository
deb_dl_dir_import() {
export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
+ rm -f "${T}"/deb_dl_dir_import.stamp
+ touch "${T}"/deb_dl_dir_import.stamp
[ ! -d "${pc}" ] && return 0
+ [ ! -d "${rootfs}"/var/cache/apt/archives/ ] && return 0
flock -s "${pc}".lock -c ' \
sudo find "${pc}" -type f -iname '*\.deb' -exec \
- cp -f --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
+ cp -n --no-preserve=owner -t "${rootfs}"/var/cache/apt/archives/ '{}' +
'
}
deb_dl_dir_export() {
export pc="${DEBDIR}/${DISTRO}/"
export rootfs="${1}"
+ export T
mkdir -p "${pc}"
flock "${pc}".lock -c ' \
- find "${rootfs}"/var/cache/apt/archives/ -type f -iname '*\.deb' |\
+ find "${rootfs}"/var/cache/apt/archives/ \
+ -cnewer "${T}"/deb_dl_dir_import.stamp \
+ -type f -iname '*\.deb' |\
while read p; do
repo_contains_package "${REPO_ISAR_DIR}"/"${DISTRO}" "${p}" && \
continue
- sudo cp -f "${p}" "${pc}"
+ sudo cp -n "${p}" "${pc}"
done
sudo chown -R $(id -u):$(id -g) "${pc}"
'
+ rm -f "${T}"/deb_dl_dir_import.stamp
}
--
2.24.1
^ permalink raw reply [flat|nested] 68+ messages in thread* Re: [PATCHv2 00/25] base-apt-rework
2019-12-16 11:49 [PATCHv2 00/25] base-apt-rework Henning Schild
` (24 preceding siblings ...)
2019-12-16 11:50 ` [PATCHv2 25/25] meta: deb-dl-dir: only export newly downloaded files Henning Schild
@ 2019-12-19 9:21 ` Henning Schild
25 siblings, 0 replies; 68+ messages in thread
From: Henning Schild @ 2019-12-19 9:21 UTC (permalink / raw)
To: isar-users; +Cc: Jan Kiszka, Baurzhan Ismagulov, chombourger
https://github.com/henning-schild-work/isar/tree/henning/staging4
On Mon, 16 Dec 2019 12:49:46 +0100
Henning Schild <henning.schild@siemens.com> wrote:
> From: Henning Schild <henning.schild@siemens.com>
>
> That one had to wait for some time. I finally managed to have a look
> at it again and came up with a v2. Sorry that one is really big and a
> lot of things happened since v1.
>
> The most significant change to last time it p23, which switches the
> way the offline cache works. Unfortunately this is turned out pretty
> big, splitting it into readably and working chunks was not reasonable.
>
> Everything up to p19 was already sent once, at least roughly ...
>
> I hope a v3 will come faster and we will have better tracking of
> changes between 2 and 3.
>
> changes since v1:
>
> - pull the base-apt building to the front of the build chain
> - performance optimze all that copying
> - add import/export of bootstrap step
> - rebased on current next
> - exclude isar-apt in deb_dl_dir_export
> - use find exec + for deb_dl_dir_import
> - fixed "==" operator in deps.sh in p8
> - fixed typo in p2 message
> - fixed variable scope to local in repository functions
>
>
> Henning Schild (25):
> repository: new class to deal with repos
> dpkg-base: add download caching of apt:// downloads
> meta: dpkg-base: convert "exit" into "return 0"
> base-apt: change the sources.list to also offer deb-src
> base-apt: add deb-src packages as well
> base-apt: do not skip gpg check when it is signed
> ci: conf: add "hello" to the sample config and every build
> meta: split all apt-get invocations into download and execution
> meta: create DL_DIR support for all apt-get downloaded .debs
> meta: import DL_DIR debs before apt-get download steps
> meta: include DL_DIR deb import/export into bootstrap
> base-apt: populate from DEBDIR as well
> base-apt: drop the "apt_cache" feature
> base-apt: do not copy debs directly out of rootfss anymore
> base-apt: rework base-apt population
> base-apt: move class "base-apt-helper" into only user
> CI: include "isar-disable-apt-cache" into all CI images
> CI: include "cowsay" into default build to test dpkg-gbp
> CI: set BB_NO_NETWORK for cached rebuild
> meta: repository: implement repo_contains_package and use it in
> base_apt
> meta: repository: simplify the matching of packages
> meta: deb-dl-dir: do not cache debs from isar-apt
> base-apt: pull base-apt population to the front of the build chain
> meta: base-apt: remove isar-apt check from population
> meta: deb-dl-dir: only export newly downloaded files
>
> doc/user_manual.md | 9 +-
> meta-isar/conf/local.conf.sample | 10 +-
> meta/classes/base-apt-helper.bbclass | 57 ----------
> meta/classes/deb-dl-dir.bbclass | 38 +++++++
> meta/classes/dpkg-base.bbclass | 47 ++++----
> meta/classes/dpkg-gbp.bbclass | 5 +
> meta/classes/dpkg.bbclass | 7 +-
> meta/classes/image-cache-extension.bbclass | 36 -------
> meta/classes/image-locales-extension.bbclass | 2 +-
> meta/classes/image-tools-extension.bbclass | 7 ++
> meta/classes/image.bbclass | 3 +-
> meta/classes/repository.bbclass | 100
> ++++++++++++++++++ meta/classes/rootfs.bbclass |
> 30 +++--- meta/conf/bitbake.conf | 6 +-
> .../isar-bootstrap/files/base-apt-sources | 1 -
> .../isar-bootstrap/isar-bootstrap.inc | 21 ++--
> meta/recipes-devtools/base-apt/base-apt.bb | 92 ++++++++++------
> .../base-apt/files/distributions.in | 3 -
> .../buildchroot/files/deps.sh | 37 ++++---
> .../isar-apt/files/distributions.in | 3 -
> meta/recipes-devtools/isar-apt/isar-apt.bb | 26 ++---
> scripts/ci_build.sh | 13 ++-
> 22 files changed, 321 insertions(+), 232 deletions(-)
> delete mode 100644 meta/classes/base-apt-helper.bbclass
> create mode 100644 meta/classes/deb-dl-dir.bbclass
> delete mode 100644 meta/classes/image-cache-extension.bbclass
> create mode 100644 meta/classes/repository.bbclass
> delete mode 100644
> meta/recipes-core/isar-bootstrap/files/base-apt-sources delete mode
> 100644 meta/recipes-devtools/base-apt/files/distributions.in delete
> mode 100644 meta/recipes-devtools/isar-apt/files/distributions.in
>
^ permalink raw reply [flat|nested] 68+ messages in thread