From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6816610675337461760 Date: Wed, 22 Apr 2020 03:35:29 -0700 (PDT) From: vijai kumar To: isar-users Message-Id: <7ceec744-6c5e-4dd3-8d33-352af3952e93@googlegroups.com> In-Reply-To: <20200422092819.49af1c46@md1za8fc.ad001.siemens.net> References: <20200417093040.15130-1-Vijaikumar_Kanagarajan@mentor.com> <20200417093040.15130-4-Vijaikumar_Kanagarajan@mentor.com> <20200422092819.49af1c46@md1za8fc.ad001.siemens.net> Subject: Re: [PATCH v5 03/13] meta: cache deb srcs as part of postprocessing MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_2799_1521946158.1587551729251" X-Google-Token: EPG7gPUFp2FFfXuDegY0 X-Google-IP: 192.94.34.34 X-TUID: 6usnBroBitQw ------=_Part_2799_1521946158.1587551729251 Content-Type: multipart/alternative; boundary="----=_Part_2800_246051232.1587551729251" ------=_Part_2800_246051232.1587551729251 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit On Wednesday, April 22, 2020 at 12:58:22 PM UTC+5:30, Henning Schild wrote: > > On Fri, 17 Apr 2020 15:00:30 +0530 > Vijai Kumar K > wrote: > > > Collect the deb sources of the corresponding deb binaries cached > > in DEBDIR as part of image postprocess. > > > > Signed-off-by: Vijai Kumar K > > > --- > > meta/classes/deb-dl-dir.bbclass | 39 > > +++++++++++++++++++++++++++++++++ meta/classes/image.bbclass | > > 2 +- meta/classes/rootfs.bbclass | 8 +++++++ > > 3 files changed, 48 insertions(+), 1 deletion(-) > > > > diff --git a/meta/classes/deb-dl-dir.bbclass > > b/meta/classes/deb-dl-dir.bbclass index 29a3d67..472b9fe 100644 > > --- a/meta/classes/deb-dl-dir.bbclass > > +++ b/meta/classes/deb-dl-dir.bbclass > > @@ -5,6 +5,45 @@ > > > > inherit repository > > > > +debsrc_download() { > > + export rootfs="$1" > > + export rootfs_distro="$2" > > + mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}" > > + ( flock 9 > > + set -e > > + printenv | grep -q BB_VERBOSE_LOGS && set -x > > + sudo -E -s <<'EOSUDO' > > + mkdir -p "${rootfs}/deb-src" > > + mountpoint -q "${rootfs}/deb-src" || \ > > + mount --bind "${DEBSRCDIR}" "${rootfs}/deb-src" > > +EOSUDO > > + find "${rootfs}/var/cache/apt/archives/" -maxdepth 1 -type f > > -iname '*\.deb' | while read package; do > > Using that cache as input is not a good idea. It does not tell you what > you have installed. It might be overfull because of the download cache, > and it might be too empty because of cache cleanups. > You probably want to iterate of the list of installed packages, or > better that manifest we have. > At mid implementation I too had that thought. Please see my comment in [1] [1] https://groups.google.com/d/msg/isar-users/quJz5ikjQPs/qLw2-FoEAwAJ > Henning > > > + local src="$( dpkg-deb --show --showformat '${Source}' > > "${package}" )" > > + # If the binary package version and source package version > > are different, then the > > + # source package version will be present inside "()" of the > > Source field. > > + local version="$( echo "$src" | cut -sd "(" -f2 | cut -sd > > ")" -f1 )" > > + if [ -z ${version} ]; then > > + version="$( dpkg-deb --show --showformat '${Version}' > > "${package}" )" > > + fi > > + # Now strip any version information that might be available. > > + src="$( echo "$src" | cut -d' ' -f1 )" > > + # If there is no source field, then the source package has > > the same name as the > > + # binary package. > > + if [ -z "${src}" ];then > > + src="$( dpkg-deb --show --showformat '${Package}' > > "${package}" )" > > + fi > > + > > + sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \ > > + sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd > > "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source > > source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" > > "${version}" > > + done > > + sudo -E -s <<'EOSUDO' > > + mountpoint -q "${rootfs}/deb-src" && \ > > + umount -l "${rootfs}/deb-src" > > + rm -rf "${rootfs}/deb-src" > > +EOSUDO > > + ) 9>"${DEBSRCDIR}/${rootfs_distro}.lock" > > +} > > + > > deb_dl_dir_import() { > > export pc="${DEBDIR}/${2}" > > export rootfs="${1}" > > diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass > > index 6b04c0a..fcaebd6 100644 > > --- a/meta/classes/image.bbclass > > +++ b/meta/classes/image.bbclass > > @@ -63,7 +63,7 @@ image_do_mounts() { > > } > > > > ROOTFSDIR = "${IMAGE_ROOTFS}" > > -ROOTFS_FEATURES += "clean-package-cache generate-manifest" > > +ROOTFS_FEATURES += "clean-package-cache generate-manifest > > cach-deb-src" ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} > > ${IMAGE_INSTALL}" ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}" > > > > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > > index cee358c..ee57989 100644 > > --- a/meta/classes/rootfs.bbclass > > +++ b/meta/classes/rootfs.bbclass > > @@ -185,6 +185,14 @@ python do_rootfs_install() { > > } > > addtask rootfs_install before do_rootfs_postprocess after do_unpack > > > > +ROOTFS_POSTPROCESS_COMMAND += > > "${@bb.utils.contains('ROOTFS_FEATURES', 'cache-deb-src', > > 'cache_deb_src', '', d)}" +cache_deb_src() { > > + rootfs_install_resolvconf > > + deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO} > > + debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO} > > + rootfs_install_clean_files > > +} > > + > > ROOTFS_POSTPROCESS_COMMAND += > > "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', > > 'rootfs_postprocess_clean_package_cache', '', d)}" > > rootfs_postprocess_clean_package_cache() { sudo -E chroot > > '${ROOTFSDIR}' \ > > ------=_Part_2800_246051232.1587551729251 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable


On Wednesday, April 22, 2020 at 12:58:22 PM UTC+5:= 30, Henning Schild wrote:
On Fr= i, 17 Apr 2020 15:00:30 +0530
Vijai Kumar K <vijaikumar...@gmail.com> wrote:

> Collect the deb sources of the corresponding deb binaries cached
> in DEBDIR as part of image postprocess.
>=20
> Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com&g= t;
> ---
> =C2=A0meta/classes/deb-dl-dir.bbclass | 39
> +++++++++++++++++++++++++++++++++ meta/classes/image.bbclass = =C2=A0 =C2=A0 =C2=A0|
> 2 +- meta/classes/rootfs.bbclass =C2=A0 =C2=A0 | =C2=A08 +++++++
> =C2=A03 files changed, 48 insertions(+), 1 deletion(-)
>=20
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index 29a3d67..472b9fe 1006= 44
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -5,6 +5,45 @@
> =C2=A0
> =C2=A0inherit repository
> =C2=A0
> +debsrc_download() {
> + =C2=A0 =C2=A0export rootfs=3D"$1"
> + =C2=A0 =C2=A0export rootfs_distro=3D"$2"
> + =C2=A0 =C2=A0mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
> + =C2=A0 =C2=A0( flock 9
> + =C2=A0 =C2=A0set -e
> + =C2=A0 =C2=A0printenv | grep -q BB_VERBOSE_LOGS && set -= x
> + =C2=A0 =C2=A0sudo -E -s <<'EOSUDO'
> + =C2=A0 =C2=A0mkdir -p "${rootfs}/deb-src"
> + =C2=A0 =C2=A0mountpoint -q "${rootfs}/deb-src" || \
> + =C2=A0 =C2=A0mount --bind "${DEBSRCDIR}" "${rootf= s}/deb-src"
> +EOSUDO
> + =C2=A0 =C2=A0find "${rootfs}/var/cache/apt/archives/&q= uot; -maxdepth 1 -type f
> -iname '*\.deb' | while read package; do

Using that cache as input is not a good idea. It does not tell you what
you have installed. It might be overfull because of the download cache,
and it might be too empty because of cache cleanups.
You probably want to iterate of the list of installed packages, or
better that manifest we have.


At mid implementation I= too had that thought. Please see my comment
in [1]
[1]=C2=A0https://groups.google.com/d/msg/isar-users/quJz5ikjQPs= /qLw2-FoEAwAJ



Henning

> + =C2=A0 =C2=A0 =C2=A0 =C2=A0local src=3D"$( dpkg-deb --show = --showformat '${Source}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# If the binary package version and s= ource package version
> are different, then the
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# source package version will be pres= ent inside "()" of the
> Source field.
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0local version=3D"$( echo "$= src" | cut -sd "(" -f2 | cut -sd
> ")" -f1 )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ -z ${version} ]; then
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0version=3D"$( dpkg= -deb --show --showformat '${Version}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0fi
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# Now strip any version information t= hat might be available.
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0src=3D"$( echo "$src" = | cut -d' ' -f1 )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# If there is no source field, then t= he source package has
> the same name as the
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# binary package.
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ -z "${src}" ];then
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0src=3D"$( dpkg-deb= --show --showformat '${Package}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0fi
> +
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo -E chroot --userspec=3D$( id -u = ):$( id -g ) ${rootfs} \
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0sh -c ' mkdir -p &q= uot;/deb-src/${1}/${2}" && cd
> "/deb-src/${1}/${2}" && apt-get -y --download-on= ly --only-source
> source "$2"=3D"$3" ' download-src "${= rootfs_distro}" "${src}"
> "${version}"
> + =C2=A0 =C2=A0done
> + =C2=A0 =C2=A0sudo -E -s <<'EOSUDO'
> + =C2=A0 =C2=A0mountpoint -q "${rootfs}/deb-src" &&a= mp; \
> + =C2=A0 =C2=A0umount -l "${rootfs}/deb-src"
> + =C2=A0 =C2=A0rm -rf "${rootfs}/deb-src"
> +EOSUDO
> + =C2=A0 =C2=A0) 9>"${DEBSRCDIR}/${rootfs_distro}.loc= k"
> +}
> +
> =C2=A0deb_dl_dir_import() {
> =C2=A0 =C2=A0 =C2=A0export pc=3D"${DEBDIR}/${2}"
> =C2=A0 =C2=A0 =C2=A0export rootfs=3D"${1}"
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbcla= ss
> index 6b04c0a..fcaebd6 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -63,7 +63,7 @@ image_do_mounts() {
> =C2=A0}
> =C2=A0
> =C2=A0ROOTFSDIR =3D "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES +=3D "clean-package-cache generate-manifest&= quot;
> +ROOTFS_FEATURES +=3D "clean-package-cache generate-manifest
> cach-deb-src" ROOTFS_PACKAGES +=3D "${IMAGE_PREINSTALL}
> ${IMAGE_INSTALL}" ROOTFS_MANIFEST_DEPLOY_DIR ?=3D "${DEP= LOY_DIR_IMAGE}"
> =C2=A0
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbc= lass
> index cee358c..ee57989 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -185,6 +185,14 @@ python do_rootfs_install() {
> =C2=A0}
> =C2=A0addtask rootfs_install before do_rootfs_postprocess after do= _unpack
> =C2=A0
> +ROOTFS_POSTPROCESS_COMMAND +=3D
> "${@bb.utils.contains('ROOTFS_FEATURES', 'ca= che-deb-src',
> 'cache_deb_src', '', d)}" +cache_deb_src() {
> + =C2=A0 =C2=A0rootfs_install_resolvconf
> + =C2=A0 =C2=A0deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
> + =C2=A0 =C2=A0debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO}
> + =C2=A0 =C2=A0rootfs_install_clean_files
> +}
> +
> =C2=A0ROOTFS_POSTPROCESS_COMMAND +=3D
> "${@bb.utils.contains('ROOTFS_FEATURES', 'cl= ean-package-cache',
> 'rootfs_postprocess_clean_package_cache', '',= d)}"
> rootfs_postprocess_clean_package_cache() { sudo -E chroot
> '${ROOTFSDIR}' \

------=_Part_2800_246051232.1587551729251-- ------=_Part_2799_1521946158.1587551729251--