From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6752900179667451904 Date: Tue, 29 Oct 2019 05:57:49 -0700 (PDT) From: chombourger@gmail.com To: isar-users Message-Id: In-Reply-To: <20191028170132.28072-3-henning.schild@siemens.com> References: <20191028170132.28072-1-henning.schild@siemens.com> <20191028170132.28072-3-henning.schild@siemens.com> Subject: Re: [PATCH 02/17] dpkg-base: add download caching of apt:// downloads MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_1714_225811669.1572353869190" X-Google-Token: EM3u4O0FzBYaSRg6U7k0 X-Google-IP: 139.181.48.2 X-TUID: dnFMBN5JlPXz ------=_Part_1714_225811669.1572353869190 Content-Type: multipart/alternative; boundary="----=_Part_1715_322508412.1572353869191" ------=_Part_1715_322508412.1572353869191 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit On Monday, October 28, 2019 at 6:01:35 PM UTC+1, Henning Schild wrote: > > From: Henning Schild > > > Store the results of "apt-get source" in the DL_DIR. This means we > really only need to fetch once on incemental rebuilds or if multiple > recipes use the same SRC_URI. > We can also later collect the results and put them in a repository. > > Signed-off-by: Henning Schild > > --- > meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++-- > meta/conf/bitbake.conf | 1 + > 2 files changed, 19 insertions(+), 2 deletions(-) > > diff --git a/meta/classes/dpkg-base.bbclass > b/meta/classes/dpkg-base.bbclass > index 3f1e99f..b39df5c 100644 > --- a/meta/classes/dpkg-base.bbclass > +++ b/meta/classes/dpkg-base.bbclass > @@ -37,14 +37,30 @@ do_apt_fetch() { > -o Dir::Etc::SourceParts="-" \ > -o APT::Get::List-Cleanup="0" > > - sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \ > - sh -c 'cd ${PP} && apt-get -y --only-source source ${SRC_APT}' > + for uri in "${SRC_APT}"; do > + sudo -E chroot --userspec=$( id -u ):$( id -g ) > ${BUILDCHROOT_DIR} \ > + sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd > /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source > source "$2"' my_script "${DISTRO}" "${uri}" > + sudo -E chroot --userspec=$( id -u ):$( id -g ) > ${BUILDCHROOT_DIR} \ > + sh -c 'cp /downloads/deb-src/"$1"/"$2"/* ${PP} && cd ${PP} && > apt-get -y --only-source source "$2"' my_script "${DISTRO}" "${uri}" > + done > + > dpkg_undo_mounts > } > > addtask apt_fetch after do_unpack before do_patch > do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock" > > +addtask cleanall_apt before do_cleanall > +do_cleanall_apt[nostamp] = "1" > +do_cleanall_apt() { > + if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then > + exit > shouldn't this be "return 0" instead? (or are we really wanting to exit the bitbake process here? > + fi > + for uri in "${SRC_APT}"; do > + rm -rf "${DEBSRCDIR}"/"${DISTRO}"/"$uri" > + done > +} > + > def get_package_srcdir(d): > s = os.path.abspath(d.getVar("S", True)) > workdir = os.path.abspath(d.getVar("WORKDIR", True)) > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index bca1114..4c0a809 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -26,6 +26,7 @@ DEPLOY_DIR = "${TMPDIR}/deploy" > FILESPATH = > "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}" > > FILE_DIRNAME = "${@os.path.dirname(d.getVar('FILE', False))}" > GITDIR = "${DL_DIR}/git" > +DEBSRCDIR = "${DL_DIR}/deb-src" > P = "${PN}-${PV}" > PF = "${PN}-${PV}-${PR}" > PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] > or 'defaultpkgname'}" > -- > 2.23.0 > > ------=_Part_1715_322508412.1572353869191 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable


On Monday, October 28, 2019 at 6:01:35 PM UTC+1, H= enning Schild wrote:
From: Henn= ing Schild <hennin...@siemens.com>

Store the results of "apt-get source" in the DL_DIR. This mea= ns we
really only need to fetch once on incemental rebuilds or if multiple
recipes use the same SRC_URI.
We can also later collect the results and put them in a repository.

Signed-off-by: Henning Schild <hennin...@siemens.com>
---
=C2=A0meta/classes/dpkg-base.bbclass | 20 ++++++++++++++++++--
=C2=A0meta/conf/bitbake.conf =C2=A0 =C2=A0 =C2=A0 =C2=A0 | =C2=A01 +
=C2=A02 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-ba= se.bbclass
index 3f1e99f..b39df5c 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -37,14 +37,30 @@ do_apt_fetch() {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0-o Dir::Etc::SourceParts=3D"-&qu= ot; \
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0-o APT::Get::List-Cleanup=3D"0&q= uot;
=C2=A0
- =C2=A0 =C2=A0sudo -E chroot --userspec=3D$( id -u ):$( id -g ) ${BUIL= DCHROOT_DIR} \
- =C2=A0 =C2=A0 =C2=A0 =C2=A0sh -c 'cd ${PP} && apt-get -y = --only-source source ${SRC_APT}'
+ =C2=A0 =C2=A0for uri in "${SRC_APT}"; do
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo -E chroot --userspec=3D$( id -u ):$( = id -g ) ${BUILDCHROOT_DIR} \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0sh -c 'mkdir -p /downloa= ds/deb-src/"$1"/"$2" && cd /downloads/deb-src/&= quot;$1"/"$2" && apt-get -y --download-only --only-s= ource source "$2"' my_script "${DISTRO}" "${ur= i}"
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo -E chroot --userspec=3D$( id -u ):$( = id -g ) ${BUILDCHROOT_DIR} \
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0sh -c 'cp /downloads/deb= -src/"$1"/"$2"/* ${PP} && cd ${PP} && a= pt-get -y --only-source source "$2"' my_script "${DISTRO= }" "${uri}"
+ =C2=A0 =C2=A0done
+
=C2=A0 =C2=A0 =C2=A0dpkg_undo_mounts
=C2=A0}
=C2=A0
=C2=A0addtask apt_fetch after do_unpack before do_patch
=C2=A0do_apt_fetch[lockfiles] +=3D "${REPO_ISAR_DIR}/isar.lock&quo= t;
=C2=A0
+addtask cleanall_apt before do_cleanall
+do_cleanall_apt[nostamp] =3D "1"
+do_cleanall_apt() {
+ =C2=A0 =C2=A0if [ -z "${@d.getVar("SRC_APT", True).str= ip()}" ]; then
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0exit

shouldn't this be "return 0&q= uot; instead?
(or are we really wanting to exit the bitbake proce= ss here?
=C2=A0
+ =C2=A0 =C2=A0fi
+ =C2=A0 =C2=A0for uri in "${SRC_APT}"; do
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0rm -rf "${DEBSRCDIR}"/"${DI= STRO}"/"$uri"
+ =C2=A0 =C2=A0done
+}
+
=C2=A0def get_package_srcdir(d):
=C2=A0 =C2=A0 =C2=A0s =3D os.path.abspath(d.getVar("S", True)= )
=C2=A0 =C2=A0 =C2=A0workdir =3D os.path.abspath(d.getVar("WOR= KDIR", True))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index bca1114..4c0a809 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -26,6 +26,7 @@ DEPLOY_DIR =3D "${TMPDIR}/deploy"
=C2=A0FILESPATH =3D "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${= P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}&qu= ot;
=C2=A0FILE_DIRNAME =3D "${@os.path.dirname(d.getVar('FILE= ', False))}"
=C2=A0GITDIR =3D "${DL_DIR}/git"
+DEBSRCDIR =3D "${DL_DIR}/deb-src"
=C2=A0P =3D "${PN}-${PV}"
=C2=A0PF =3D "${PN}-${PV}-${PR}"
=C2=A0PN =3D "${@bb.parse.BBHandler.vars_from_file(d.getVar(&= #39;FILE', False),d)[0] or 'defaultpkgname'}"
--=20
2.23.0

------=_Part_1715_322508412.1572353869191-- ------=_Part_1714_225811669.1572353869190--