From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 6816610675337461760 Date: Wed, 22 Apr 2020 02:57:13 -0700 (PDT) From: vijai kumar To: isar-users Message-Id: In-Reply-To: <20200422090639.645bf93b@md1za8fc.ad001.siemens.net> References: <20200417093040.15130-1-Vijaikumar_Kanagarajan@mentor.com> <20200417093040.15130-4-Vijaikumar_Kanagarajan@mentor.com> <20200422090639.645bf93b@md1za8fc.ad001.siemens.net> Subject: Re: [PATCH v5 03/13] meta: cache deb srcs as part of postprocessing MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_2018_1853211499.1587549433635" X-Google-Token: EPmpgPUFKHi_OMjuoWM0 X-Google-IP: 192.94.34.34 X-TUID: edOcmnR839pE ------=_Part_2018_1853211499.1587549433635 Content-Type: multipart/alternative; boundary="----=_Part_2019_467622940.1587549433635" ------=_Part_2019_467622940.1587549433635 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit On Wednesday, April 22, 2020 at 12:36:42 PM UTC+5:30, Henning Schild wrote: > > On Fri, 17 Apr 2020 15:00:30 +0530 > Vijai Kumar K > wrote: > > > Collect the deb sources of the corresponding deb binaries cached > > in DEBDIR as part of image postprocess. > > > > Signed-off-by: Vijai Kumar K > > > --- > > meta/classes/deb-dl-dir.bbclass | 39 > > +++++++++++++++++++++++++++++++++ meta/classes/image.bbclass | > > 2 +- meta/classes/rootfs.bbclass | 8 +++++++ > > 3 files changed, 48 insertions(+), 1 deletion(-) > > > > diff --git a/meta/classes/deb-dl-dir.bbclass > > b/meta/classes/deb-dl-dir.bbclass index 29a3d67..472b9fe 100644 > > --- a/meta/classes/deb-dl-dir.bbclass > > +++ b/meta/classes/deb-dl-dir.bbclass > > @@ -5,6 +5,45 @@ > > > > inherit repository > > > > +debsrc_download() { > > + export rootfs="$1" > > + export rootfs_distro="$2" > > + mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}" > > + ( flock 9 > > I think you can grab that lock only for the actual writes, and keep the > generation of the list out of the critical section. > To note, this lock also guards the mount part. > Note that i played with this "flock 9" syntax instead of what is used in > deb-dl-dir, it did not work as expected. Probably because it will be > many shells and 9 is a different fd in all of them. > Interesting. Works as expected here. If we still need to switch the syntax to be sure, we could. > + set -e > > + printenv | grep -q BB_VERBOSE_LOGS && set -x > > + sudo -E -s <<'EOSUDO' > > + mkdir -p "${rootfs}/deb-src" > > + mountpoint -q "${rootfs}/deb-src" || \ > > + mount --bind "${DEBSRCDIR}" "${rootfs}/deb-src" > > +EOSUDO > > + find "${rootfs}/var/cache/apt/archives/" -maxdepth 1 -type f > > -iname '*\.deb' | while read package; do > > + local src="$( dpkg-deb --show --showformat '${Source}' > > "${package}" )" > > + # If the binary package version and source package version > > are different, then the > > + # source package version will be present inside "()" of the > > Source field. > > dpkg-query(1) > > dpkg-deb --show --showformat '${source:Version}' > dpkg-deb --show --showformat '${source:Upstream-Version}' > > might help to write this cleaner. > Thanks. Will use this. > > > + local version="$( echo "$src" | cut -sd "(" -f2 | cut -sd > > ")" -f1 )" > > + if [ -z ${version} ]; then > > + version="$( dpkg-deb --show --showformat '${Version}' > > "${package}" )" > > + fi > > + # Now strip any version information that might be available. > > + src="$( echo "$src" | cut -d' ' -f1 )" > > + # If there is no source field, then the source package has > > the same name as the > > + # binary package. > > + if [ -z "${src}" ];then > > + src="$( dpkg-deb --show --showformat '${Package}' > > "${package}" )" > > + fi > > I still could not find those proxies that all downloading functions > need in their env. > >>From what I see, the rootfs class from where this is called, already takes care of this https://github.com/ilbers/isar/blob/next/meta/classes/rootfs.bbclass#L22 > > Henning > > > + sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \ > > + sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd > > "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source > > source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" > > "${version}" > > + done > > + sudo -E -s <<'EOSUDO' > > + mountpoint -q "${rootfs}/deb-src" && \ > > + umount -l "${rootfs}/deb-src" > > + rm -rf "${rootfs}/deb-src" > > +EOSUDO > > + ) 9>"${DEBSRCDIR}/${rootfs_distro}.lock" > > +} > > + > > deb_dl_dir_import() { > > export pc="${DEBDIR}/${2}" > > export rootfs="${1}" > > diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass > > index 6b04c0a..fcaebd6 100644 > > --- a/meta/classes/image.bbclass > > +++ b/meta/classes/image.bbclass > > @@ -63,7 +63,7 @@ image_do_mounts() { > > } > > > > ROOTFSDIR = "${IMAGE_ROOTFS}" > > -ROOTFS_FEATURES += "clean-package-cache generate-manifest" > > +ROOTFS_FEATURES += "clean-package-cache generate-manifest > > cach-deb-src" ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} > > ${IMAGE_INSTALL}" ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}" > > > > diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass > > index cee358c..ee57989 100644 > > --- a/meta/classes/rootfs.bbclass > > +++ b/meta/classes/rootfs.bbclass > > @@ -185,6 +185,14 @@ python do_rootfs_install() { > > } > > addtask rootfs_install before do_rootfs_postprocess after do_unpack > > > > +ROOTFS_POSTPROCESS_COMMAND += > > "${@bb.utils.contains('ROOTFS_FEATURES', 'cache-deb-src', > > 'cache_deb_src', '', d)}" +cache_deb_src() { > > + rootfs_install_resolvconf > > + deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO} > > + debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO} > > + rootfs_install_clean_files > > +} > > + > > ROOTFS_POSTPROCESS_COMMAND += > > "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', > > 'rootfs_postprocess_clean_package_cache', '', d)}" > > rootfs_postprocess_clean_package_cache() { sudo -E chroot > > '${ROOTFSDIR}' \ > > ------=_Part_2019_467622940.1587549433635 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable


On Wednesday, April 22, 2020 at 12:36:42 PM UTC+5:= 30, Henning Schild wrote:
On Fr= i, 17 Apr 2020 15:00:30 +0530
Vijai Kumar K <vijaikumar...@gmail.com> wrote:

> Collect the deb sources of the corresponding deb binaries cached
> in DEBDIR as part of image postprocess.
>=20
> Signed-off-by: Vijai Kumar K <Vijaikumar_...@mentor.com&g= t;
> ---
> =C2=A0meta/classes/deb-dl-dir.bbclass | 39
> +++++++++++++++++++++++++++++++++ meta/classes/image.bbclass = =C2=A0 =C2=A0 =C2=A0|
> 2 +- meta/classes/rootfs.bbclass =C2=A0 =C2=A0 | =C2=A08 +++++++
> =C2=A03 files changed, 48 insertions(+), 1 deletion(-)
>=20
> diff --git a/meta/classes/deb-dl-dir.bbclass
> b/meta/classes/deb-dl-dir.bbclass index 29a3d67..472b9fe 1006= 44
> --- a/meta/classes/deb-dl-dir.bbclass
> +++ b/meta/classes/deb-dl-dir.bbclass
> @@ -5,6 +5,45 @@
> =C2=A0
> =C2=A0inherit repository
> =C2=A0
> +debsrc_download() {
> + =C2=A0 =C2=A0export rootfs=3D"$1"
> + =C2=A0 =C2=A0export rootfs_distro=3D"$2"
> + =C2=A0 =C2=A0mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
> + =C2=A0 =C2=A0( flock 9

I think you can grab that lock only for the actual writes, and keep the
generation of the list out of the critical section.=C2=A0

To note, this lock also guards the mount part.=C2=A0=
=C2=A0
Note th= at i played with this "flock 9" syntax instead of what is used in
deb-dl-dir, it did not work as expected. Probably because it will be
many shells and 9 is a different fd in all of them.

Interesting. Works as expected here.= =C2=A0 If we still need to switch the syntax
to be sure, we could= .

> + = =C2=A0 =C2=A0set -e
> + =C2=A0 =C2=A0printenv | grep -q BB_VERBOSE_LOGS && set -= x
> + =C2=A0 =C2=A0sudo -E -s <<'EOSUDO'
> + =C2=A0 =C2=A0mkdir -p "${rootfs}/deb-src"
> + =C2=A0 =C2=A0mountpoint -q "${rootfs}/deb-src" || \
> + =C2=A0 =C2=A0mount --bind "${DEBSRCDIR}" "${rootf= s}/deb-src"
> +EOSUDO
> + =C2=A0 =C2=A0find "${rootfs}/var/cache/apt/archives/&q= uot; -maxdepth 1 -type f
> -iname '*\.deb' | while read package; do
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0local src=3D"$( dpkg-deb --show = --showformat '${Source}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# If the binary package version and s= ource package version
> are different, then the
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# source package version will be pres= ent inside "()" of the
> Source field.

dpkg-query(1)

dpkg-deb --show --showformat '${source:Version}'
dpkg-deb --show --showformat '${source:Upstream-Version}'

might help to write this cleaner.

Thanks. Will use this.
=C2= =A0

> + =C2=A0 =C2=A0 =C2=A0 =C2=A0local version=3D"$( echo "$= src" | cut -sd "(" -f2 | cut -sd
> ")" -f1 )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ -z ${version} ]; then
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0version=3D"$( dpkg= -deb --show --showformat '${Version}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0fi
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# Now strip any version information t= hat might be available.
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0src=3D"$( echo "$src" = | cut -d' ' -f1 )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# If there is no source field, then t= he source package has
> the same name as the
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0# binary package.
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if [ -z "${src}" ];then
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0src=3D"$( dpkg-deb= --show --showformat '${Package}'
> "${package}" )"
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0fi

I still could not find those proxies that all downloading functions
need in their env.

From what I see, the rootfs class from= where this is called, already takes care of this

=C2=A0

Henning

> + =C2=A0 =C2=A0 =C2=A0 =C2=A0sudo -E chroot --userspec=3D$( id -u = ):$( id -g ) ${rootfs} \
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0sh -c ' mkdir -p &q= uot;/deb-src/${1}/${2}" && cd
> "/deb-src/${1}/${2}" && apt-get -y --download-on= ly --only-source
> source "$2"=3D"$3" ' download-src "${= rootfs_distro}" "${src}"
> "${version}"
> + =C2=A0 =C2=A0done
> + =C2=A0 =C2=A0sudo -E -s <<'EOSUDO'
> + =C2=A0 =C2=A0mountpoint -q "${rootfs}/deb-src" &&a= mp; \
> + =C2=A0 =C2=A0umount -l "${rootfs}/deb-src"
> + =C2=A0 =C2=A0rm -rf "${rootfs}/deb-src"
> +EOSUDO
> + =C2=A0 =C2=A0) 9>"${DEBSRCDIR}/${rootfs_distro}.loc= k"
> +}
> +
> =C2=A0deb_dl_dir_import() {
> =C2=A0 =C2=A0 =C2=A0export pc=3D"${DEBDIR}/${2}"
> =C2=A0 =C2=A0 =C2=A0export rootfs=3D"${1}"
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbcla= ss
> index 6b04c0a..fcaebd6 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -63,7 +63,7 @@ image_do_mounts() {
> =C2=A0}
> =C2=A0
> =C2=A0ROOTFSDIR =3D "${IMAGE_ROOTFS}"
> -ROOTFS_FEATURES +=3D "clean-package-cache generate-manifest&= quot;
> +ROOTFS_FEATURES +=3D "clean-package-cache generate-manifest
> cach-deb-src" ROOTFS_PACKAGES +=3D "${IMAGE_PREINSTALL}
> ${IMAGE_INSTALL}" ROOTFS_MANIFEST_DEPLOY_DIR ?=3D "${DEP= LOY_DIR_IMAGE}"
> =C2=A0
> diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbc= lass
> index cee358c..ee57989 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -185,6 +185,14 @@ python do_rootfs_install() {
> =C2=A0}
> =C2=A0addtask rootfs_install before do_rootfs_postprocess after do= _unpack
> =C2=A0
> +ROOTFS_POSTPROCESS_COMMAND +=3D
> "${@bb.utils.contains('ROOTFS_FEATURES', 'ca= che-deb-src',
> 'cache_deb_src', '', d)}" +cache_deb_src() {
> + =C2=A0 =C2=A0rootfs_install_resolvconf
> + =C2=A0 =C2=A0deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
> + =C2=A0 =C2=A0debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO}
> + =C2=A0 =C2=A0rootfs_install_clean_files
> +}
> +
> =C2=A0ROOTFS_POSTPROCESS_COMMAND +=3D
> "${@bb.utils.contains('ROOTFS_FEATURES', 'cl= ean-package-cache',
> 'rootfs_postprocess_clean_package_cache', '',= d)}"
> rootfs_postprocess_clean_package_cache() { sudo -E chroot
> '${ROOTFSDIR}' \

------=_Part_2019_467622940.1587549433635-- ------=_Part_2018_1853211499.1587549433635--