From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6771003123014041600 Date: Sat, 25 Jan 2020 06:06:59 -0800 (PST) From: vijai kumar To: isar-users Message-Id: <8bb53ef9-64ee-449e-a3b2-58eda2dab41e@googlegroups.com> In-Reply-To: <20191216115011.17664-10-henning.schild@siemens.com> References: <20191216115011.17664-1-henning.schild@siemens.com> <20191216115011.17664-10-henning.schild@siemens.com> Subject: Re: [PATCHv2 09/25] meta: create DL_DIR support for all apt-get downloaded .debs MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_3595_1541819702.1579961219160" X-Google-Token: EIOXsfEFXLiGwFe_GQ00 X-Google-IP: 192.94.34.34 X-TUID: K5CUs16TqGOI ------=_Part_3595_1541819702.1579961219160 Content-Type: multipart/alternative; boundary="----=_Part_3596_492889660.1579961219161" ------=_Part_3596_492889660.1579961219161 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit On Monday, December 16, 2019 at 5:20:15 PM UTC+5:30, Henning Schild wrote: > > From: Henning Schild > > > Hook in between the "apt-get --download-only"s and the "apt-get"s and > copy out all the debs for later. > > Signed-off-by: Henning Schild > > --- > meta/classes/deb-dl-dir.bbclass | 15 +++++++++++++++ > meta/classes/dpkg-base.bbclass | 1 + > meta/classes/dpkg-gbp.bbclass | 1 + > meta/classes/dpkg.bbclass | 1 + > meta/classes/image-locales-extension.bbclass | 2 +- > meta/classes/image-tools-extension.bbclass | 1 + > meta/classes/rootfs.bbclass | 12 ++++++++++-- > meta/conf/bitbake.conf | 1 + > 8 files changed, 31 insertions(+), 3 deletions(-) > create mode 100644 meta/classes/deb-dl-dir.bbclass > > diff --git a/meta/classes/deb-dl-dir.bbclass > b/meta/classes/deb-dl-dir.bbclass > new file mode 100644 > index 0000000..e41e981 > --- /dev/null > +++ b/meta/classes/deb-dl-dir.bbclass > @@ -0,0 +1,15 @@ > +# This software is a part of ISAR. > +# Copyright (C) 2019 Siemens AG > +# > +# SPDX-License-Identifier: MIT > + > +deb_dl_dir_export() { > + export pc="${DEBDIR}/${DISTRO}" > + export rootfs="${1}" > + mkdir -p "${pc}" > + flock "${pc}".lock -c ' \ > + sudo find "${rootfs}"/var/cache/apt/archives/ -type f -iname > '*\.deb' \ > + -exec cp -f '{}' "${pc}" \; > + sudo chown -R $(id -u):$(id -g) "${pc}" > + ' > +} > diff --git a/meta/classes/dpkg-base.bbclass > b/meta/classes/dpkg-base.bbclass > index aa11660..fd0b07f 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -8,6 +8,7 @@ inherit buildchroot > inherit debianize > inherit terminal > inherit repository > +inherit deb-dl-dir > > DEPENDS ?= "" > > diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass > index 2afe9db..7e908eb 100644 > --- a/meta/classes/dpkg-gbp.bbclass > +++ b/meta/classes/dpkg-gbp.bbclass > @@ -15,6 +15,7 @@ do_install_builddeps_append() { > sudo -E chroot ${BUILDCHROOT_DIR} \ > apt-get install -y -o Debug::pkgProblemResolver=yes \ > --no-install-recommends --download-only > ${GBP_DEPENDS} > + deb_dl_dir_export "${BUILDCHROOT_DIR}" > sudo -E chroot ${BUILDCHROOT_DIR} \ > apt-get install -y -o Debug::pkgProblemResolver=yes \ > --no-install-recommends ${GBP_DEPENDS} > diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass > index b26c645..7e2c86d 100644 > --- a/meta/classes/dpkg.bbclass > +++ b/meta/classes/dpkg.bbclass > @@ -9,6 +9,7 @@ do_install_builddeps() { While testing this series in a downstream project, I do not see a part of KBUILD_DEPENDS cached. We don't have the "linux-custom recipe rework" changes by Cedric there. The old Linux recipe overrides this task with its own variant. I have added deb_dl_* there as well to fix the issue. This might become a problem when more such recipes pop-up. Unlikely, but possible. Recipe authors should know they have to take care of this if they decide to override do_install_builddeps. Thanks, Vijai Kumar K E="${@ isar_export_proxies(d)}" > sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ > ${PP}/${PPS} ${DISTRO_ARCH} --download-only > + deb_dl_dir_export "${BUILDCHROOT_DIR}" > sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \ > ${PP}/${PPS} ${DISTRO_ARCH} > dpkg_undo_mounts > diff --git a/meta/classes/image-locales-extension.bbclass > b/meta/classes/image-locales-extension.bbclass > index 3c0758f..0f0d0ca 100644 > --- a/meta/classes/image-locales-extension.bbclass > +++ b/meta/classes/image-locales-extension.bbclass > @@ -25,7 +25,7 @@ def get_nopurge(d): > j.split()[0].split(".")[0], > j.split()[0])))) > > -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN += > "image_install_localepurge_download" > +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT += > "image_install_localepurge_download" > image_install_localepurge_download[weight] = "40" > image_install_localepurge_download() { > sudo -E chroot '${ROOTFSDIR}' \ > diff --git a/meta/classes/image-tools-extension.bbclass > b/meta/classes/image-tools-extension.bbclass > index b8531cb..ec67d94 100644 > --- a/meta/classes/image-tools-extension.bbclass > +++ b/meta/classes/image-tools-extension.bbclass > @@ -34,6 +34,7 @@ do_install_imager_deps() { > --allow-unauthenticated --allow-downgrades --download-only > install \ > ${IMAGER_INSTALL}' > > + deb_dl_dir_export ${BUILDCHROOT_DIR} > sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \ > apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends > -y \ > --allow-unauthenticated --allow-downgrades install \ > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > index b4dbfa3..a6b6110 100644 > --- a/meta/classes/rootfs.bbclass > +++ b/meta/classes/rootfs.bbclass > @@ -1,6 +1,8 @@ > # This software is a part of ISAR. > # Copyright (c) Siemens AG, 2019 > > +inherit deb-dl-dir > + > ROOTFS_ARCH ?= "${DISTRO_ARCH}" > ROOTFS_DISTRO ?= "${DISTRO}" > ROOTFS_PACKAGES ?= "" > @@ -119,8 +121,14 @@ rootfs_install_pkgs_download() { > /usr/bin/apt-get ${ROOTFS_APT_ARGS} --download-only > ${ROOTFS_PACKAGES} > } > > -ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??= "" > -ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN}" > +ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??= "" > +ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}" > + > +ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache" > +rootfs_export_package_cache[weight] = "5" > +rootfs_export_package_cache() { > + deb_dl_dir_export ${ROOTFSDIR} > +} > > ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if > (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}" > rootfs_install_clean_files[weight] = "2" > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index ab608da..d86c5b9 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy" > FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}" > FILESEXTRAPATHS ?= "__default:" > GITDIR = "${DL_DIR}/git" > +DEBDIR = "${DL_DIR}/deb" > DEBSRCDIR = "${DL_DIR}/deb-src" > P = "${PN}-${PV}" > PF = "${PN}-${PV}-${PR}" > -- > 2.24.1 > > ------=_Part_3596_492889660.1579961219161 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable


On Monday, December 16, 2019 at 5:20:15 PM UTC+5:3= 0, Henning Schild wrote:
From: = Henning Schild <hennin...@siemens.com>

Hook in between the "apt-get --download-only"s and the "= apt-get"s and
copy out all the debs for later.

Signed-off-by: Henning Schild <hennin...@siemens.com>
---
=C2=A0meta/classes/deb-dl-dir.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0| 15 +++++++++++++++
=C2=A0meta/classes/dpkg-base.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 | =C2=A01 +
=C2=A0meta/classes/dpkg-gbp.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0| =C2=A01 +
=C2=A0meta/classes/dpkg.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| =C2=A01 +
=C2=A0meta/classes/image-locales-extension.bbclass | =C2=A02 +-
=C2=A0meta/classes/image-tools-extension.bbclass =C2=A0 | =C2=A01 = +
=C2=A0meta/classes/rootfs.bbclass =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0| 12 ++++++++++--
=C2=A0meta/conf/bitbake.conf =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 | =C2=A01 +
=C2=A08 files changed, 31 insertions(+), 3 deletions(-)
=C2=A0create mode 100644 meta/classes/deb-dl-dir.bbclass

diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl= -dir.bbclass
new file mode 100644
index 0000000..e41e981
--- /dev/null
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -0,0 +1,15 @@
+# This software is a part of ISAR.
+# Copyright (C) 2019 Siemens AG
+#
+# SPDX-License-Identifier: MIT
+
+deb_dl_dir_export() {
+ =C2=A0 =C2=A0export pc=3D"${DEBDIR}/${DISTRO}"
+ =C2=A0 =C2=A0export rootfs=3D"${1}"
+ =C2=A0 =C2=A0mkdir -p "${pc}"
+ =C2=A0 =C2=A0flock "${pc}".lock -c ' \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo find "${rootfs}"/var/cache/= apt/archives/ -type f -iname '*\.deb' \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0-exec cp -f '{}' &qu= ot;${pc}" \;
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo chown -R $(id -u):$(id -g) "${pc= }"
+ =C2=A0 =C2=A0'
+}
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-ba= se.bbclass
index aa11660..fd0b07f 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -8,6 +8,7 @@ inherit buildchroot
=C2=A0inherit debianize
=C2=A0inherit terminal
=C2=A0inherit repository
+inherit deb-dl-dir
=C2=A0
=C2=A0DEPENDS ?=3D ""
=C2=A0
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp= .bbclass
index 2afe9db..7e908eb 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -15,6 +15,7 @@ do_install_builddeps_append() {
=C2=A0 =C2=A0 =C2=A0sudo -E chroot ${BUILDCHROOT_DIR} \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0apt-get install -y -o Debug::pkgProbl= emResolver=3Dyes \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0--no-install-recommends --download-only ${GBP_DEPENDS}
+ =C2=A0 =C2=A0deb_dl_dir_export "${BUILDCHROOT_DIR}"
=C2=A0 =C2=A0 =C2=A0sudo -E chroot ${BUILDCHROOT_DIR} \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0apt-get install -y -o Debug::pkgProbl= emResolver=3Dyes \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index b26c645..7e2c86d 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,6 +9,7 @@ do_install_builddeps() {
=C2=A0
<= div>
While testing this series in a downstream project, I do not s= ee a part of KBUILD_DEPENDS cached.

We don't have the "linu= x-custom recipe rework" changes by Cedric there. The old Linux recipe = overrides
this task with its own variant.

I have added deb_dl_* = there as well to fix the issue.

This might become a problem when mor= e such recipes pop-up. Unlikely, but possible.
Recipe authors should kno= w they have to take care of this if they decide to override
do_install_= builddeps.


Thanks,
V= ijai Kumar K


=C2=A0 =C2=A0 =C2=A0E=3D"${@ isar_export_proxies(d)}&q= uot;
=C2=A0 =C2=A0 =C2=A0sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0${PP}/${PPS} ${DISTRO_ARCH} --downloa= d-only
+ =C2=A0 =C2=A0deb_dl_dir_export "${BUILDCHROOT_DIR}"
=C2=A0 =C2=A0 =C2=A0sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0${PP}/${PPS} ${DISTRO_ARCH}
=C2=A0 =C2=A0 =C2=A0dpkg_undo_mounts
diff --git a/meta/classes/image-locales-extension.bbclass b/meta/c= lasses/image-locales-extension.bbclass
index 3c0758f..0f0d0ca 100644
--- a/meta/classes/image-locales-extension.bbclass
+++ b/meta/classes/image-locales-extension.bbclass
@@ -25,7 +25,7 @@ def get_nopurge(d):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0j.split()[0].split(".")[0],
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0j.split()[0]))))
=C2=A0
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN +=3D "image_install_loca= lepurge_download"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT +=3D "image_install_loc= alepurge_download"
=C2=A0image_install_localepurge_download[weight] =3D "40"= ;
=C2=A0image_install_localepurge_download() {
=C2=A0 =C2=A0 =C2=A0sudo -E chroot '${ROOTFSDIR}' \
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/cla= sses/image-tools-extension.bbclass
index b8531cb..ec67d94 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -34,6 +34,7 @@ do_install_imager_deps() {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0--allow-unauthenticated= --allow-downgrades --download-only install \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0${IMAGER_INSTALL}'
=C2=A0
+ =C2=A0 =C2=A0deb_dl_dir_export ${BUILDCHROOT_DIR}
=C2=A0 =C2=A0 =C2=A0sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0apt-get -o Debug::pkgProblemResolver= =3Dyes --no-install-recommends -y \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0--allow-unauthenticated= --allow-downgrades install \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index b4dbfa3..a6b6110 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -1,6 +1,8 @@
=C2=A0# This software is a part of ISAR.
=C2=A0# Copyright (c) Siemens AG, 2019
=C2=A0
+inherit deb-dl-dir
+
=C2=A0ROOTFS_ARCH ?=3D "${DISTRO_ARCH}"
=C2=A0ROOTFS_DISTRO ?=3D "${DISTRO}"
=C2=A0ROOTFS_PACKAGES ?=3D ""
@@ -119,8 +121,14 @@ rootfs_install_pkgs_download() {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0/usr/bin/apt-get ${ROOTFS_APT_ARGS} -= -download-only ${ROOTFS_PACKAGES}
=C2=A0}
=C2=A0
-ROOTFS_INSTALL_COMMAND_BEFORE_CLEAN ??=3D ""
-ROOTFS_INSTALL_COMMAND +=3D "${ROOTFS_INSTALL_COMMAND_BEFORE= _CLEAN}"
+ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT ??=3D ""
+ROOTFS_INSTALL_COMMAND +=3D "${ROOTFS_INSTALL_COMMAND_BEFORE= _EXPORT}"
+
+ROOTFS_INSTALL_COMMAND +=3D "rootfs_export_package_cache"
+rootfs_export_package_cache[weight] =3D "5"
+rootfs_export_package_cache() {
+ =C2=A0 =C2=A0deb_dl_dir_export ${ROOTFSDIR}
+}
=C2=A0
=C2=A0ROOTFS_INSTALL_COMMAND +=3D "${@ 'rootfs_install_clean_f= iles' if (d.getVar('ROOTFS_CLEAN_FILES') or '').st= rip() else ''}"
=C2=A0rootfs_install_clean_files[weight] =3D "2"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index ab608da..d86c5b9 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR =3D "${TMPDIR}/deploy"
=C2=A0FILE_DIRNAME =3D "${@os.path.dirname(d.getVar('FILE= ', False))}"
=C2=A0FILESEXTRAPATHS ?=3D "__default:"
=C2=A0GITDIR =3D "${DL_DIR}/git"
+DEBDIR =3D "${DL_DIR}/deb"
=C2=A0DEBSRCDIR =3D "${DL_DIR}/deb-src"
=C2=A0P =3D "${PN}-${PV}"
=C2=A0PF =3D "${PN}-${PV}-${PR}"
--=20
2.24.1

------=_Part_3596_492889660.1579961219161-- ------=_Part_3595_1541819702.1579961219160--