* [PATCH v9 01/14] rootfs: Make rootfs finalize a separate task
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
@ 2020-11-26 11:18 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 02/14] deb-dl-dir: Cache host distro debs separately Vijai Kumar K
` (6 subsequent siblings)
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:18 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
With the current implementation it is difficult to append a
postprocess function which requires a chroot environment.
For example, to add a postprocess function which runs apt-get to
download all source of packages installed in the target.
rootfs_postprocess_finalize is not actually an optional feature
but instead a necessary cleanup function for image class.
So, move the implementation to image class and make it as a task.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/image.bbclass | 41 ++++++++++++++++++++++++++++++++++++-
meta/classes/rootfs.bbclass | 40 ------------------------------------
2 files changed, 40 insertions(+), 41 deletions(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index a296cc0..2391529 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -63,7 +63,7 @@ image_do_mounts() {
}
ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "clean-package-cache finalize-rootfs generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
@@ -173,5 +173,44 @@ python do_deploy() {
}
addtask deploy before do_build after do_image
+do_rootfs_finalize() {
+ sudo -s <<'EOSUDO'
+ test -e "${ROOTFSDIR}/chroot-setup.sh" && \
+ "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
+ rm -f "${ROOTFSDIR}/chroot-setup.sh"
+
+ test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
+ find "${ROOTFSDIR}/usr/bin" \
+ -maxdepth 1 -name 'qemu-*-static' -type f -delete
+
+ mountpoint -q '${ROOTFSDIR}/isar-apt' && \
+ umount -l ${ROOTFSDIR}/isar-apt
+ rmdir --ignore-fail-on-non-empty ${ROOTFSDIR}/isar-apt
+
+ mountpoint -q '${ROOTFSDIR}/base-apt' && \
+ umount -l ${ROOTFSDIR}/base-apt
+ rmdir --ignore-fail-on-non-empty ${ROOTFSDIR}/base-apt
+
+ mountpoint -q '${ROOTFSDIR}/dev' && \
+ umount -l ${ROOTFSDIR}/dev
+ mountpoint -q '${ROOTFSDIR}/sys' && \
+ umount -l ${ROOTFSDIR}/proc
+ mountpoint -q '${ROOTFSDIR}/sys' && \
+ umount -l ${ROOTFSDIR}/sys
+
+ rm -f "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
+
+ rm -f "${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list"
+ rm -f "${ROOTFSDIR}/etc/apt/preferences.d/isar-apt"
+ rm -f "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
+
+ mv "${ROOTFSDIR}/etc/apt/sources-list" \
+ "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+
+ rm -f "${ROOTFSDIR}/etc/apt/sources-list"
+EOSUDO
+}
+addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
+
# Last so that the image type can overwrite tasks if needed
inherit ${IMAGE_TYPE}
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 2c5d0dc..f9c8995 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -11,7 +11,6 @@ ROOTFS_PACKAGES ?= ""
# available features are:
# 'clean-package-cache' - delete package cache from rootfs
# 'generate-manifest' - generate a package manifest of the rootfs into ${ROOTFS_MANIFEST_DEPLOY_DIR}
-# 'finalize-rootfs' - delete files needed to chroot into the rootfs
ROOTFS_FEATURES ?= ""
ROOTFS_APT_ARGS="install --yes -o Debug::pkgProblemResolver=yes"
@@ -201,45 +200,6 @@ rootfs_generate_manifest () {
${ROOTFS_MANIFEST_DEPLOY_DIR}/"${PF}".manifest
}
-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'finalize-rootfs', 'rootfs_postprocess_finalize', '', d)}"
-rootfs_postprocess_finalize() {
- sudo -s <<'EOSUDO'
- test -e "${ROOTFSDIR}/chroot-setup.sh" && \
- "${ROOTFSDIR}/chroot-setup.sh" "cleanup" "${ROOTFSDIR}"
- rm -f "${ROOTFSDIR}/chroot-setup.sh"
-
- test ! -e "${ROOTFSDIR}/usr/share/doc/qemu-user-static" && \
- find "${ROOTFSDIR}/usr/bin" \
- -maxdepth 1 -name 'qemu-*-static' -type f -delete
-
- mountpoint -q '${ROOTFSDIR}/isar-apt' && \
- umount -l ${ROOTFSDIR}/isar-apt
- rmdir --ignore-fail-on-non-empty ${ROOTFSDIR}/isar-apt
-
- mountpoint -q '${ROOTFSDIR}/base-apt' && \
- umount -l ${ROOTFSDIR}/base-apt
- rmdir --ignore-fail-on-non-empty ${ROOTFSDIR}/base-apt
-
- mountpoint -q '${ROOTFSDIR}/dev' && \
- umount -l ${ROOTFSDIR}/dev
- mountpoint -q '${ROOTFSDIR}/sys' && \
- umount -l ${ROOTFSDIR}/proc
- mountpoint -q '${ROOTFSDIR}/sys' && \
- umount -l ${ROOTFSDIR}/sys
-
- rm -f "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
-
- rm -f "${ROOTFSDIR}/etc/apt/sources.list.d/isar-apt.list"
- rm -f "${ROOTFSDIR}/etc/apt/preferences.d/isar-apt"
- rm -f "${ROOTFSDIR}/etc/apt/sources.list.d/base-apt.list"
-
- mv "${ROOTFSDIR}/etc/apt/sources-list" \
- "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
-
- rm -f "${ROOTFSDIR}/etc/apt/sources-list"
-EOSUDO
-}
-
do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
python do_rootfs_postprocess() {
# Take care that its correctly mounted:
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 02/14] deb-dl-dir: Cache host distro debs separately
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
2020-11-26 11:18 ` [PATCH v9 01/14] rootfs: Make rootfs finalize a separate task Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 03/14] meta: cache deb srcs as part of postprocessing Vijai Kumar K
` (5 subsequent siblings)
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
In case of targets where HOST_DISTRO!=DISTRO, like rpi-stretch,
we were still caching the debs from both the distros into the same
DEBDIR/DISTRO directory. With this change, HOST_DISTRO would be
cached in the relevant subdirectory and avoids mixing of debs from
two separate distros.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 4 ++--
meta/classes/dpkg-gbp.bbclass | 8 ++++++--
meta/classes/dpkg.bbclass | 8 ++++++--
meta/classes/image-tools-extension.bbclass | 4 ++--
meta/classes/rootfs.bbclass | 4 ++--
meta/recipes-core/isar-bootstrap/isar-bootstrap.inc | 12 ++++++++++--
6 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 54c2994..92f0259 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -6,7 +6,7 @@
inherit repository
deb_dl_dir_import() {
- export pc="${DEBDIR}/${DISTRO}/"
+ export pc="${DEBDIR}/${2}"
export rootfs="${1}"
[ ! -d "${pc}" ] && return 0
sudo mkdir -p "${rootfs}"/var/cache/apt/archives/
@@ -20,7 +20,7 @@ deb_dl_dir_import() {
}
deb_dl_dir_export() {
- export pc="${DEBDIR}/${DISTRO}/"
+ export pc="${DEBDIR}/${2}"
export rootfs="${1}"
mkdir -p "${pc}"
flock "${pc}".lock -c '
diff --git a/meta/classes/dpkg-gbp.bbclass b/meta/classes/dpkg-gbp.bbclass
index afa1e19..ba5c3eb 100644
--- a/meta/classes/dpkg-gbp.bbclass
+++ b/meta/classes/dpkg-gbp.bbclass
@@ -12,11 +12,15 @@ GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
do_install_builddeps_append() {
dpkg_do_mounts
- deb_dl_dir_import "${BUILDCHROOT_DIR}"
+ distro="${DISTRO}"
+ if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
+ distro="${HOST_DISTRO}"
+ fi
+ deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends --download-only ${GBP_DEPENDS}
- deb_dl_dir_export "${BUILDCHROOT_DIR}"
+ deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}"
sudo -E chroot ${BUILDCHROOT_DIR} \
apt-get install -y -o Debug::pkgProblemResolver=yes \
--no-install-recommends ${GBP_DEPENDS}
diff --git a/meta/classes/dpkg.bbclass b/meta/classes/dpkg.bbclass
index a24386d..4e7c2f7 100644
--- a/meta/classes/dpkg.bbclass
+++ b/meta/classes/dpkg.bbclass
@@ -9,10 +9,14 @@ PACKAGE_ARCH ?= "${DISTRO_ARCH}"
do_install_builddeps() {
dpkg_do_mounts
E="${@ isar_export_proxies(d)}"
- deb_dl_dir_import "${BUILDCHROOT_DIR}"
+ distro="${DISTRO}"
+ if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
+ distro="${HOST_DISTRO}"
+ fi
+ deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${PACKAGE_ARCH} --download-only
- deb_dl_dir_export "${BUILDCHROOT_DIR}"
+ deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}"
sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
${PP}/${PPS} ${PACKAGE_ARCH}
dpkg_undo_mounts
diff --git a/meta/classes/image-tools-extension.bbclass b/meta/classes/image-tools-extension.bbclass
index 6590ee7..0b067ff 100644
--- a/meta/classes/image-tools-extension.bbclass
+++ b/meta/classes/image-tools-extension.bbclass
@@ -25,7 +25,7 @@ do_install_imager_deps() {
buildchroot_do_mounts
E="${@ isar_export_proxies(d)}"
- deb_dl_dir_import ${BUILDCHROOT_DIR}
+ deb_dl_dir_import ${BUILDCHROOT_DIR} ${DISTRO}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get update \
-o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
@@ -35,7 +35,7 @@ do_install_imager_deps() {
--allow-unauthenticated --allow-downgrades --download-only install \
${IMAGER_INSTALL}'
- deb_dl_dir_export ${BUILDCHROOT_DIR}
+ deb_dl_dir_export ${BUILDCHROOT_DIR} ${DISTRO}
sudo -E chroot ${BUILDCHROOT_DIR} sh -c ' \
apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y \
--allow-unauthenticated --allow-downgrades install \
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index f9c8995..fb4a3b6 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -114,7 +114,7 @@ rootfs_install_resolvconf() {
ROOTFS_INSTALL_COMMAND += "rootfs_import_package_cache"
rootfs_import_package_cache[weight] = "5"
rootfs_import_package_cache() {
- deb_dl_dir_import ${ROOTFSDIR}
+ deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
}
ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
@@ -131,7 +131,7 @@ ROOTFS_INSTALL_COMMAND += "${ROOTFS_INSTALL_COMMAND_BEFORE_EXPORT}"
ROOTFS_INSTALL_COMMAND += "rootfs_export_package_cache"
rootfs_export_package_cache[weight] = "5"
rootfs_export_package_cache() {
- deb_dl_dir_export ${ROOTFSDIR}
+ deb_dl_dir_export ${ROOTFSDIR} ${ROOTFS_DISTRO}
}
ROOTFS_INSTALL_COMMAND += "${@ 'rootfs_install_clean_files' if (d.getVar('ROOTFS_CLEAN_FILES') or '').strip() else ''}"
diff --git a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
index 9a5ebfd..6b2e5a7 100644
--- a/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
+++ b/meta/recipes-core/isar-bootstrap/isar-bootstrap.inc
@@ -258,7 +258,11 @@ isar_bootstrap() {
export IS_HOST debootstrap_args E
if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
sudo rm -rf --one-file-system "${ROOTFSDIR}"
- deb_dl_dir_import "${ROOTFSDIR}"
+ if [ "${IS_HOST}" ];then
+ deb_dl_dir_import "${ROOTFSDIR}" "${HOST_DISTRO}"
+ else
+ deb_dl_dir_import "${ROOTFSDIR}" "${DISTRO}"
+ fi
sudo -E -s <<'EOSUDO'
set -e
@@ -358,7 +362,11 @@ isar_bootstrap() {
ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
EOSUDO
fi
- deb_dl_dir_export "${ROOTFSDIR}"
+ if [ "${IS_HOST}" ];then
+ deb_dl_dir_export "${ROOTFSDIR}" "${HOST_DISTRO}"
+ else
+ deb_dl_dir_export "${ROOTFSDIR}" "${DISTRO}"
+ fi
}
CLEANFUNCS = "clean_deploy"
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 03/14] meta: cache deb srcs as part of postprocessing
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
2020-11-26 11:18 ` [PATCH v9 01/14] rootfs: Make rootfs finalize a separate task Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 02/14] deb-dl-dir: Cache host distro debs separately Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 04/14] deb-dl-dir: Make debsrc_download faster Vijai Kumar K
` (4 subsequent siblings)
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Collect the deb sources of the corresponding deb binaries cached
in DEBDIR as part of image postprocess.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 27 +++++++++++++++++++++++++++
meta/classes/image.bbclass | 2 +-
meta/classes/rootfs.bbclass | 8 ++++++++
3 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 92f0259..bbc6f0a 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -5,6 +5,33 @@
inherit repository
+debsrc_download() {
+ export rootfs="$1"
+ export rootfs_distro="$2"
+ mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
+ sudo -E -s <<'EOSUDO'
+ mkdir -p "${rootfs}/deb-src"
+ mountpoint -q "${rootfs}/deb-src" || \
+ mount --bind "${DEBSRCDIR}" "${rootfs}/deb-src"
+EOSUDO
+ ( flock 9
+ set -e
+ printenv | grep -q BB_VERBOSE_LOGS && set -x
+ find "${rootfs}/var/cache/apt/archives/" -maxdepth 1 -type f -iname '*\.deb' | while read package; do
+ local src="$( dpkg-deb --show --showformat '${source:Package}' "${package}" )"
+ local version="$( dpkg-deb --show --showformat '${source:Version}' "${package}" )"
+
+ sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \
+ sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
+ done
+ ) 9>"${DEBSRCDIR}/${rootfs_distro}.lock"
+ sudo -E -s <<'EOSUDO'
+ mountpoint -q "${rootfs}/deb-src" && \
+ umount -l "${rootfs}/deb-src"
+ rm -rf "${rootfs}/deb-src"
+EOSUDO
+}
+
deb_dl_dir_import() {
export pc="${DEBDIR}/${2}"
export rootfs="${1}"
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 2391529..51cb2e1 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -63,7 +63,7 @@ image_do_mounts() {
}
ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "clean-package-cache generate-manifest"
+ROOTFS_FEATURES += "clean-package-cache generate-manifest cache-deb-src"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index fb4a3b6..f972a89 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -184,6 +184,14 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack
+ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'cache-deb-src', 'cache_deb_src', '', d)}"
+cache_deb_src() {
+ rootfs_install_resolvconf
+ deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
+ debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO}
+ rootfs_install_clean_files
+}
+
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
rootfs_postprocess_clean_package_cache() {
sudo -E chroot '${ROOTFSDIR}' \
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 04/14] deb-dl-dir: Make debsrc_download faster
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
` (2 preceding siblings ...)
2020-11-26 11:19 ` [PATCH v9 03/14] meta: cache deb srcs as part of postprocessing Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 05/14] deb-dl-dir: Download files only belonging to the current image Vijai Kumar K
` (3 subsequent siblings)
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Eventhough apt-get source skips redownloading of files, it is still
slow and takes a lot of time. Instead, lookup if the dsc file is already
present in the cache and skip based on it.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 2 ++
1 file changed, 2 insertions(+)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index bbc6f0a..0e2cef2 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -20,6 +20,8 @@ EOSUDO
find "${rootfs}/var/cache/apt/archives/" -maxdepth 1 -type f -iname '*\.deb' | while read package; do
local src="$( dpkg-deb --show --showformat '${source:Package}' "${package}" )"
local version="$( dpkg-deb --show --showformat '${source:Version}' "${package}" )"
+ local dscfile=$(find "${DEBSRCDIR}"/"${rootfs_distro}" -name "${src}_${version}.dsc")
+ [ -n "$dscfile" ] && continue
sudo -E chroot --userspec=$( id -u ):$( id -g ) ${rootfs} \
sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 05/14] deb-dl-dir: Download files only belonging to the current image
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
` (3 preceding siblings ...)
2020-11-26 11:19 ` [PATCH v9 04/14] deb-dl-dir: Make debsrc_download faster Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 06/14] deb-dl-dir: Factor out the mounting part Vijai Kumar K
` (2 subsequent siblings)
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Avoid downloading deb-srcs for debs cached from other image builds.
One way to ensure that is to see if the package is present in the dpkg
status file.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 0e2cef2..410befd 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -5,6 +5,15 @@
inherit repository
+is_not_part_of_current_build() {
+ local package="$( dpkg-deb --show --showformat '${Package}' "${1}" )"
+ local output="$( grep -hs "^Package: ${package}" \
+ "${IMAGE_ROOTFS}"/var/lib/dpkg/status \
+ "${BUILDCHROOT_HOST_DIR}"/var/lib/dpkg/status \
+ "${BUILDCHROOT_TARGET_DIR}"/var/lib/dpkg/status )"
+ [ -z "${output}" ]
+}
+
debsrc_download() {
export rootfs="$1"
export rootfs_distro="$2"
@@ -18,6 +27,7 @@ EOSUDO
set -e
printenv | grep -q BB_VERBOSE_LOGS && set -x
find "${rootfs}/var/cache/apt/archives/" -maxdepth 1 -type f -iname '*\.deb' | while read package; do
+ is_not_part_of_current_build "${package}" && continue
local src="$( dpkg-deb --show --showformat '${source:Package}' "${package}" )"
local version="$( dpkg-deb --show --showformat '${source:Version}' "${package}" )"
local dscfile=$(find "${DEBSRCDIR}"/"${rootfs_distro}" -name "${src}_${version}.dsc")
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 06/14] deb-dl-dir: Factor out the mounting part
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
` (4 preceding siblings ...)
2020-11-26 11:19 ` [PATCH v9 05/14] deb-dl-dir: Download files only belonging to the current image Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 07/14] deb-dl-dir: Fix skipping of removed files Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 08/14] repository: Add a sanity test to check missing sources Vijai Kumar K
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Factor out the mount and unmount section to separate function for
easy readability. No functional change intended.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 32 ++++++++++++++++++++++----------
1 file changed, 22 insertions(+), 10 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 410befd..64d1823 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -14,15 +14,30 @@ is_not_part_of_current_build() {
[ -z "${output}" ]
}
+debsrc_do_mounts() {
+ sudo -s <<EOSUDO
+ mkdir -p "${1}/deb-src"
+ mountpoint -q "${1}/deb-src" || \
+ mount --bind "${DEBSRCDIR}" "${1}/deb-src"
+EOSUDO
+}
+
+debsrc_undo_mounts() {
+ sudo -s <<EOSUDO
+ mkdir -p "${1}/deb-src"
+ mountpoint -q "${1}/deb-src" && \
+ umount -l "${1}/deb-src"
+ rm -rf "${1}/deb-src"
+EOSUDO
+}
+
debsrc_download() {
export rootfs="$1"
export rootfs_distro="$2"
mkdir -p "${DEBSRCDIR}"/"${rootfs_distro}"
- sudo -E -s <<'EOSUDO'
- mkdir -p "${rootfs}/deb-src"
- mountpoint -q "${rootfs}/deb-src" || \
- mount --bind "${DEBSRCDIR}" "${rootfs}/deb-src"
-EOSUDO
+
+ debsrc_do_mounts "${rootfs}"
+
( flock 9
set -e
printenv | grep -q BB_VERBOSE_LOGS && set -x
@@ -37,11 +52,8 @@ EOSUDO
sh -c ' mkdir -p "/deb-src/${1}/${2}" && cd "/deb-src/${1}/${2}" && apt-get -y --download-only --only-source source "$2"="$3" ' download-src "${rootfs_distro}" "${src}" "${version}"
done
) 9>"${DEBSRCDIR}/${rootfs_distro}.lock"
- sudo -E -s <<'EOSUDO'
- mountpoint -q "${rootfs}/deb-src" && \
- umount -l "${rootfs}/deb-src"
- rm -rf "${rootfs}/deb-src"
-EOSUDO
+
+ debsrc_undo_mounts "${rootfs}"
}
deb_dl_dir_import() {
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 07/14] deb-dl-dir: Fix skipping of removed files
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
` (5 preceding siblings ...)
2020-11-26 11:19 ` [PATCH v9 06/14] deb-dl-dir: Factor out the mounting part Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:19 ` [PATCH v9 08/14] repository: Add a sanity test to check missing sources Vijai Kumar K
7 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Some packages are installed and then removed, like localepurge from
image-locales-extension.bbclass. Those information would not be
available in dpkg status file. Use dpkg log instead to see if the
package has been used on the target.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/deb-dl-dir.bbclass | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/meta/classes/deb-dl-dir.bbclass b/meta/classes/deb-dl-dir.bbclass
index 64d1823..29bf45b 100644
--- a/meta/classes/deb-dl-dir.bbclass
+++ b/meta/classes/deb-dl-dir.bbclass
@@ -7,10 +7,16 @@ inherit repository
is_not_part_of_current_build() {
local package="$( dpkg-deb --show --showformat '${Package}' "${1}" )"
- local output="$( grep -hs "^Package: ${package}" \
- "${IMAGE_ROOTFS}"/var/lib/dpkg/status \
- "${BUILDCHROOT_HOST_DIR}"/var/lib/dpkg/status \
- "${BUILDCHROOT_TARGET_DIR}"/var/lib/dpkg/status )"
+ local arch="$( dpkg-deb --show --showformat '${Architecture}' "${1}" )"
+ local version="$( dpkg-deb --show --showformat '${Version}' "${1}" )"
+ # Since we are parsing all the debs in DEBDIR, we can to some extend
+ # try to eliminate some debs that are not part of the current multiconfig
+ # build using the below method.
+ local output="$( grep -hs "status installed ${package}:${arch} ${version}" \
+ "${IMAGE_ROOTFS}"/var/log/dpkg.log \
+ "${BUILDCHROOT_HOST_DIR}"/var/log/dpkg.log \
+ "${BUILDCHROOT_TARGET_DIR}"/var/log/dpkg.log | head -1 )"
+
[ -z "${output}" ]
}
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 08/14] repository: Add a sanity test to check missing sources
2020-11-26 11:18 [PATCH v9 00/14] Debsrc caching Vijai Kumar K
` (6 preceding siblings ...)
2020-11-26 11:19 ` [PATCH v9 07/14] deb-dl-dir: Fix skipping of removed files Vijai Kumar K
@ 2020-11-26 11:19 ` Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES Vijai Kumar K
7 siblings, 1 reply; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:19 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Add a sanity test routine to test if sources for all packages
are present in the repo.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/repository.bbclass | 10 ++++++++++
meta/recipes-devtools/base-apt/base-apt.bb | 2 ++
2 files changed, 12 insertions(+)
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index c70138f..7b6e47b 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -96,3 +96,13 @@ repo_contains_package() {
# no
return 2
}
+
+repo_sanity_test() {
+ local dir="$1"
+ local dbdir="$2"
+ local output="$( reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing )"
+ if [ -n "${output}" ]; then
+ bbwarn "One or more sources are missing in base-apt"
+ bbwarn "${output}"
+ fi
+}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index da3e32e..8940ae8 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -66,6 +66,8 @@ repo() {
fi
populate_base_apt
+ repo_sanity_test "${REPO_BASE_DIR}"/"${BASE_DISTRO}" \
+ "${REPO_BASE_DB_DIR}"/"${BASE_DISTRO}"
}
python do_cache() {
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES
2020-11-26 11:19 ` [PATCH v9 08/14] repository: Add a sanity test to check missing sources Vijai Kumar K
@ 2020-11-26 11:21 ` Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 10/14] repository: Fix failures due to missing section Vijai Kumar K
` (3 more replies)
0 siblings, 4 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:21 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Eventhough we are collecting the debsrcs as part of postprocess,
it could not be considered a ROOTFS_FEATURE, instead a base-apt one.
Introduce BASE_REPO_FEATURES, to provide user with control to enable
or disable cache-deb-src. Disabled by default, since it is not required
for normal offline build to work.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta-isar/conf/local.conf.sample | 4 ++++
meta/classes/image.bbclass | 4 +++-
meta/classes/repository.bbclass | 9 +++++----
meta/classes/rootfs.bbclass | 1 -
meta/recipes-devtools/base-apt/base-apt.bb | 1 +
5 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/meta-isar/conf/local.conf.sample b/meta-isar/conf/local.conf.sample
index ddd8cda..a191a8b 100644
--- a/meta-isar/conf/local.conf.sample
+++ b/meta-isar/conf/local.conf.sample
@@ -180,6 +180,10 @@ ISAR_CROSS_COMPILE ?= "0"
# NOTE: this works for amd64 and arm64 targets so far
#ISAR_ENABLE_COMPAT_ARCH ?= "1"
+# Uncomment this to enable caching of all source packages.
+# Without this feature, only sources of packages downloaded with apt:// are downloaded.
+#BASE_REPO_FEATURES ?= "cache-deb-src"
+
#
# Uncomment this to enable use of cached base repository
#ISAR_USE_CACHED_BASE_REPO ?= "1"
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 51cb2e1..5c2c88c 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -63,10 +63,12 @@ image_do_mounts() {
}
ROOTFSDIR = "${IMAGE_ROOTFS}"
-ROOTFS_FEATURES += "clean-package-cache generate-manifest cache-deb-src"
+ROOTFS_FEATURES += "clean-package-cache generate-manifest"
ROOTFS_PACKAGES += "${IMAGE_PREINSTALL} ${IMAGE_INSTALL}"
ROOTFS_MANIFEST_DEPLOY_DIR ?= "${DEPLOY_DIR_IMAGE}"
+ROOTFS_POSTPROCESS_COMMAND_prepend = "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'cache_deb_src', '', d)} "
+
inherit rootfs
inherit image-sdk-extension
inherit image-tools-extension
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 7b6e47b..1f475dc 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -100,9 +100,10 @@ repo_contains_package() {
repo_sanity_test() {
local dir="$1"
local dbdir="$2"
- local output="$( reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing )"
- if [ -n "${output}" ]; then
- bbwarn "One or more sources are missing in base-apt"
- bbwarn "${output}"
+ if [ "${@bb.utils.contains('BASE_REPO_FEATURES', 'cache-deb-src', 'yes', 'no', d)}" = "yes" ];then
+ local output="$( reprepro -s -b "${dir}" --dbdir "${dbdir}" sourcemissing )"
+ if [ -n "${output}" ]; then
+ bbfatal "One or more sources are missing in repo. ${output}"
+ fi
fi
}
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index f972a89..9142f12 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -184,7 +184,6 @@ python do_rootfs_install() {
}
addtask rootfs_install before do_rootfs_postprocess after do_unpack
-ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'cache-deb-src', 'cache_deb_src', '', d)}"
cache_deb_src() {
rootfs_install_resolvconf
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
diff --git a/meta/recipes-devtools/base-apt/base-apt.bb b/meta/recipes-devtools/base-apt/base-apt.bb
index 8940ae8..506a28f 100644
--- a/meta/recipes-devtools/base-apt/base-apt.bb
+++ b/meta/recipes-devtools/base-apt/base-apt.bb
@@ -10,6 +10,7 @@ SRC_URI = "file://distributions.in"
BASE_REPO_KEY ?= ""
KEYFILES ?= ""
+BASE_REPO_FEATURES ?= ""
populate_base_apt() {
find "${DEBDIR}"/"${DISTRO}" -name '*\.deb' | while read package; do
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 10/14] repository: Fix failures due to missing section
2020-11-26 11:21 ` [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES Vijai Kumar K
@ 2020-11-26 11:21 ` Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 11/14] scripts/ci_build.sh: Enable deb-src caching Vijai Kumar K
` (2 subsequent siblings)
3 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:21 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
reprepro's includedsc failed for certain packages like makedev while
complaining about missing Section information. This information is
optional according to Debian[1]. Use a dummy value to avoid such
failures.
[1] https://wiki.debian.org/DebianRepository/Format
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/repository.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/repository.bbclass b/meta/classes/repository.bbclass
index 1f475dc..ac395db 100644
--- a/meta/classes/repository.bbclass
+++ b/meta/classes/repository.bbclass
@@ -42,7 +42,7 @@ repo_add_srcpackage() {
if [ -n "${GNUPGHOME}" ]; then
export GNUPGHOME="${GNUPGHOME}"
fi
- reprepro -b "${dir}" --dbdir "${dbdir}" -C main -P source \
+ reprepro -b "${dir}" --dbdir "${dbdir}" -C main -S - -P source \
includedsc "${codename}" \
"$@"
}
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 11/14] scripts/ci_build.sh: Enable deb-src caching
2020-11-26 11:21 ` [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 10/14] repository: Fix failures due to missing section Vijai Kumar K
@ 2020-11-26 11:21 ` Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 12/14] rootfs: Fix possible overwrite of existing resolv.conf Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 13/14] rootfs: Handle failures when postprocess is rerun Vijai Kumar K
3 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:21 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Enable debsrc caching for CI builds.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
scripts/ci_build.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh
index c0a032e..fb458ae 100755
--- a/scripts/ci_build.sh
+++ b/scripts/ci_build.sh
@@ -184,6 +184,7 @@ if [ -n "$REPRO_BUILD" ]; then
sed -i -e 's/^BB_NO_NETWORK/#BB_NO_NETWORK/g' conf/local.conf
fi
+sed -i -e 's/^#BASE_REPO_FEATURES ?= "cache-deb-src"/BASE_REPO_FEATURES ?= "cache-deb-src"/g' conf/local.conf
# Start cross build for the defined set of configurations
sed -i -e 's/ISAR_CROSS_COMPILE ?= "0"/ISAR_CROSS_COMPILE ?= "1"/g' conf/local.conf
bitbake $BB_ARGS $CROSS_TARGETS_SET
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 12/14] rootfs: Fix possible overwrite of existing resolv.conf
2020-11-26 11:21 ` [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 10/14] repository: Fix failures due to missing section Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 11/14] scripts/ci_build.sh: Enable deb-src caching Vijai Kumar K
@ 2020-11-26 11:21 ` Vijai Kumar K
2020-11-26 11:21 ` [PATCH v9 13/14] rootfs: Handle failures when postprocess is rerun Vijai Kumar K
3 siblings, 0 replies; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:21 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
There is a possiblilty that one of the packages installed in the
rootfs provides /etc/resolv.conf and we might accidentally remove
it. Fix it by taking a backup of any existing resolv conf files
and restoring it later. This is needed since we could not effectively
move caching before rootfs_install_clean_files since we need the
latest dpkg log.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/rootfs.bbclass | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index 9142f12..b6dc2c8 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -185,10 +185,18 @@ python do_rootfs_install() {
addtask rootfs_install before do_rootfs_postprocess after do_unpack
cache_deb_src() {
+ if [ -e "${ROOTFSDIR}"/etc/resolv.conf ]; then
+ sudo mv "${ROOTFSDIR}"/etc/resolv.conf "${ROOTFSDIR}"/etc/resolv.conf.isar
+ fi
rootfs_install_resolvconf
+
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO}
- rootfs_install_clean_files
+
+ sudo rm -f "${ROOTFSDIR}"/etc/resolv.conf
+ if [ -e "${ROOTFSDIR}"/etc/resolv.conf.isar ]; then
+ sudo mv "${ROOTFSDIR}"/etc/resolv.conf.isar "${ROOTFSDIR}"/etc/resolv.conf
+ fi
}
ROOTFS_POSTPROCESS_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'clean-package-cache', 'rootfs_postprocess_clean_package_cache', '', d)}"
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 13/14] rootfs: Handle failures when postprocess is rerun
2020-11-26 11:21 ` [PATCH v9 09/14] base-apt: Introduce BASE_REPO_FEATURES Vijai Kumar K
` (2 preceding siblings ...)
2020-11-26 11:21 ` [PATCH v9 12/14] rootfs: Fix possible overwrite of existing resolv.conf Vijai Kumar K
@ 2020-11-26 11:21 ` Vijai Kumar K
2020-11-26 11:22 ` [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching Vijai Kumar K
3 siblings, 1 reply; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:21 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
The apt state information in var/lib/apt/lists is cleared as
part of postprocessing. This makes apt-get calls in cache_deb_src
fail when rerunning the postprocess task.
Since we cannot run apt-get update again to refresh the state
information, copy the apt state information from the initial
bootstrapped image.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
meta/classes/rootfs.bbclass | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/meta/classes/rootfs.bbclass b/meta/classes/rootfs.bbclass
index b6dc2c8..f7ba4c8 100644
--- a/meta/classes/rootfs.bbclass
+++ b/meta/classes/rootfs.bbclass
@@ -189,6 +189,10 @@ cache_deb_src() {
sudo mv "${ROOTFSDIR}"/etc/resolv.conf "${ROOTFSDIR}"/etc/resolv.conf.isar
fi
rootfs_install_resolvconf
+ # Note: ISAR updates the apt state information(apt-get update) only once during bootstrap and
+ # relies on that through out the build. Copy that state information instead of apt-get update
+ # which generates a new state from upstream.
+ sudo cp -Trpn "${BOOTSTRAP_SRC}/var/lib/apt/lists/" "${ROOTFSDIR}/var/lib/apt/lists/"
deb_dl_dir_import ${ROOTFSDIR} ${ROOTFS_DISTRO}
debsrc_download ${ROOTFSDIR} ${ROOTFS_DISTRO}
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching
2020-11-26 11:21 ` [PATCH v9 13/14] rootfs: Handle failures when postprocess is rerun Vijai Kumar K
@ 2020-11-26 11:22 ` Vijai Kumar K
2020-11-26 17:28 ` Anton Mikanovich
0 siblings, 1 reply; 18+ messages in thread
From: Vijai Kumar K @ 2020-11-26 11:22 UTC (permalink / raw)
To: isar-users, ibr; +Cc: Vijai Kumar K
Add details about deb-src caching.
Signed-off-by: Vijai Kumar K <Vijaikumar_Kanagarajan@mentor.com>
---
doc/user_manual.md | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/doc/user_manual.md b/doc/user_manual.md
index dd6fc34..8827b29 100644
--- a/doc/user_manual.md
+++ b/doc/user_manual.md
@@ -919,3 +919,33 @@ And build the corresponding image target:
```
bitbake mc:qemuarm64-buster:isar-image-base
```
+## Cache all upstream Debian source packages in local apt
+
+### Motivation
+
+OSS license compliance: Some licenses require to provide the corresponding sources code,
+other require copyright attributions that may be best provided via the source code. In
+addition, you may want to archive the code locally in order to ensure reproducibility (and
+modifiability) in the future.
+
+Currently the local-apt generated has only Debian binary packages. Extend the local-apt
+to have Debian source packages as well.
+
+### Solution
+
+ - Trigger download of Debian source packages as part of rootfs postprocess.
+
+With the current base-apt implementation, we already cache all the binary packages that
+we download and install onto the target rootfs and buildchroot. This is then used to
+generate a local-apt for offline build.
+
+Use rootfs postprocessing to parse through the the list fo deb files in ${DEDDIR} and
+download the corresponding Debian source file using "apt-get source" command.
+This caches the sources of all the Debian packages that are downloaded and installed onto
+the target rootfs and buildchroots.
+
+By default, the Debian source caching is not enabled.
+To enable it, add the below line to your local.conf file.
+```
+BASE_REPO_FEATURES = "cache-deb-src"
+```
--
2.17.1
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching
2020-11-26 11:22 ` [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching Vijai Kumar K
@ 2020-11-26 17:28 ` Anton Mikanovich
2020-11-26 17:56 ` vijaikumar....@gmail.com
0 siblings, 1 reply; 18+ messages in thread
From: Anton Mikanovich @ 2020-11-26 17:28 UTC (permalink / raw)
To: isar-users
test
26.11.2020 14:22, Vijai Kumar K wrote:
> +Use rootfs postprocessing to parse through the the list fo deb files in ${DEDDIR} and
> +download the corresponding Debian source file using "apt-get source" command.
There are typos, probably should be 'through the list of deb'.
--
Anton Mikanovich
Promwad Ltd.
External service provider of ilbers GmbH
Maria-Merian-Str. 8
85521 Ottobrunn, Germany
+49 (89) 122 67 24-0
Commercial register Munich, HRB 214197
General Manager: Baurzhan Ismagulov
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching
2020-11-26 17:28 ` Anton Mikanovich
@ 2020-11-26 17:56 ` vijaikumar....@gmail.com
2020-11-26 18:12 ` vijaikumar....@gmail.com
0 siblings, 1 reply; 18+ messages in thread
From: vijaikumar....@gmail.com @ 2020-11-26 17:56 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 727 bytes --]
On Thursday, November 26, 2020 at 10:58:54 PM UTC+5:30 ami...@ilbers.de
wrote:
> test
>
> 26.11.2020 14:22, Vijai Kumar K wrote:
> > +Use rootfs postprocessing to parse through the the list fo deb files in
> ${DEDDIR} and
> > +download the corresponding Debian source file using "apt-get source"
> command.
> There are typos, probably should be 'through the list of deb'.
>
Yes. Is it possible for you to amend it while applying this series?
Thanks,
Vijai Kumar K
>
> --
> Anton Mikanovich
> Promwad Ltd.
> External service provider of ilbers GmbH
> Maria-Merian-Str. 8
> 85521 Ottobrunn, Germany
> +49 (89) 122 67 24-0
> Commercial register Munich, HRB 214197
> General Manager: Baurzhan Ismagulov
>
>
[-- Attachment #1.2: Type: text/html, Size: 1171 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [PATCH v9 14/14] doc/user_manual.md: Document details about deb-src caching
2020-11-26 17:56 ` vijaikumar....@gmail.com
@ 2020-11-26 18:12 ` vijaikumar....@gmail.com
0 siblings, 0 replies; 18+ messages in thread
From: vijaikumar....@gmail.com @ 2020-11-26 18:12 UTC (permalink / raw)
To: isar-users
[-- Attachment #1.1: Type: text/plain, Size: 948 bytes --]
On Thursday, November 26, 2020 at 11:26:20 PM UTC+5:30
vijaikumar....@gmail.com wrote:
> On Thursday, November 26, 2020 at 10:58:54 PM UTC+5:30 ami...@ilbers.de
> wrote:
>
>> test
>>
>> 26.11.2020 14:22, Vijai Kumar K wrote:
>> > +Use rootfs postprocessing to parse through the the list fo deb files
>> in ${DEDDIR} and
>> > +download the corresponding Debian source file using "apt-get source"
>> command.
>> There are typos, probably should be 'through the list of deb'.
>>
>
> Yes. Is it possible for you to amend it while applying this series?
>
Nevermind. I am resending this patch. its also DEBDIR not DEDDIR.
Thanks,
Vijai Kumar K
>
> Thanks,
> Vijai Kumar K
>
>
>>
>> --
>> Anton Mikanovich
>> Promwad Ltd.
>> External service provider of ilbers GmbH
>> Maria-Merian-Str. 8
>> 85521 Ottobrunn, Germany
>> +49 (89) 122 67 24-0
>> Commercial register Munich, HRB 214197
>> General Manager: Baurzhan Ismagulov
>>
>>
[-- Attachment #1.2: Type: text/html, Size: 1898 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread