* [PATCH v3] Convert apt source fetcher into native bitbake variant
@ 2024-11-29 14:05 'Jan Kiszka' via isar-users
2024-12-06 7:30 ` Uladzimir Bely
0 siblings, 1 reply; 2+ messages in thread
From: 'Jan Kiszka' via isar-users @ 2024-11-29 14:05 UTC (permalink / raw)
To: isar-users; +Cc: Uladzimir Bely, Moessbauer, Felix (T CED SES-DE)
From: Jan Kiszka <jan.kiszka@siemens.com>
There is no major functional difference, but we no longer have to
manipulate SRC_URI by registering an official fetcher for apt://.
As the fetching no longer takes place in separate tasks, do_fetch and
do_unpack need to gain the extra flags that were so far assigned to
apt_fetch and apt_unpack. That happens conditionally, i.e. only if
SRC_URI actually contains an apt URL.
One difference to the original version is the possibility - even if
practically of minor relevance - to unpack multiple apt sources into S.
The old version contained a loop but was directing dpkg-source to a
pre-existing dir which would have failed on the second iteration. The
new version now folds the results together after each step.
There is no progress report realized because dpkg-source unfortunately
does not provide information upfront to make this predictable, thus
expressible in form of percentage.
Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
Changes in v3:
- keep split by distro codename in downloads/deb-src
meta/classes/dpkg-base.bbclass | 104 ++++-----------------------------
meta/lib/aptsrc_fetcher.py | 88 ++++++++++++++++++++++++++++
2 files changed, 99 insertions(+), 93 deletions(-)
create mode 100644 meta/lib/aptsrc_fetcher.py
diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-base.bbclass
index b4ea8e17..c02c07a8 100644
--- a/meta/classes/dpkg-base.bbclass
+++ b/meta/classes/dpkg-base.bbclass
@@ -79,110 +79,28 @@ do_adjust_git[lockfiles] += "${DL_DIR}/git/isar.lock"
inherit patch
addtask patch after do_adjust_git
-SRC_APT ?= ""
-
-# filter out all "apt://" URIs out of SRC_URI and stick them into SRC_APT
python() {
- src_uri = (d.getVar('SRC_URI', False) or "").split()
+ from bb.fetch2 import methods
- prefix = "apt://"
- src_apt = []
- for u in src_uri:
- if u.startswith(prefix):
- src_apt.append(u[len(prefix) :])
- d.setVar('SRC_URI:remove', u)
+ # apt-src fetcher
+ import aptsrc_fetcher
+ methods.append(aptsrc_fetcher.AptSrc())
- d.prependVar('SRC_APT', ' '.join(src_apt))
+ src_uri = (d.getVar('SRC_URI', False) or "").split()
+ for u in src_uri:
+ if u.startswith("apt://"):
+ d.appendVarFlag('do_fetch', 'depends', d.getVar('SCHROOT_DEP'))
- if len(d.getVar('SRC_APT').strip()) > 0:
- bb.build.addtask('apt_unpack', 'do_patch', '', d)
- bb.build.addtask('cleanall_apt', 'do_cleanall', '', d)
+ d.appendVarFlag('do_unpack', 'cleandirs', d.getVar('S'))
+ d.setVarFlag('do_unpack', 'network', d.getVar('TASK_USE_SUDO'))
+ break
# container docker fetcher
import container_fetcher
- from bb.fetch2 import methods
methods.append(container_fetcher.Container())
}
-do_apt_fetch() {
- E="${@ isar_export_proxies(d)}"
- schroot_create_configs
-
- session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
- echo "Started session: ${session_id}"
-
- schroot_cleanup() {
- schroot -q -f -e -c ${session_id} > /dev/null 2>&1
- schroot_delete_configs
- }
- trap 'exit 1' INT HUP QUIT TERM ALRM USR1
- trap 'schroot_cleanup' EXIT
-
- schroot -r -c ${session_id} -d / -u root -- \
- rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
- schroot -r -c ${session_id} -d / -- \
- sh -c '
- set -e
- for uri in $2; do
- mkdir -p /downloads/deb-src/"$1"/${uri}
- cd /downloads/deb-src/"$1"/${uri}
- apt-get -y --download-only --only-source source ${uri}
- done' \
- my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${SRC_APT}"
-
- schroot -e -c ${session_id}
- schroot_delete_configs
-}
-
-addtask apt_fetch
-do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
-do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
-
-# Add dependency from the correct schroot: host or target
-do_apt_fetch[depends] += "${SCHROOT_DEP}"
-
-do_apt_unpack() {
- rm -rf ${S}
- schroot_create_configs
-
- session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
- echo "Started session: ${session_id}"
-
- schroot_cleanup() {
- schroot -q -f -e -c ${session_id} > /dev/null 2>&1
- schroot_delete_configs
- }
- trap 'exit 1' INT HUP QUIT TERM ALRM USR1
- trap 'schroot_cleanup' EXIT
-
- schroot -r -c ${session_id} -d / -u root -- \
- rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
- schroot -r -c ${session_id} -d / -- \
- sh -c '
- set -e
- for uri in $2; do
- dscfile="$(apt-get -y -qq --print-uris --only-source source $uri | cut -d " " -f2 | grep -E "*.dsc")"
- cd ${PP}
- cp /downloads/deb-src/"${1}"/${uri}/* ${PP}
- dpkg-source -x "${dscfile}" "${PPS}"
- done' \
- my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}" "${SRC_APT}"
-
- schroot -e -c ${session_id}
- schroot_delete_configs
-}
-do_apt_unpack[network] = "${TASK_USE_SUDO}"
-
-addtask apt_unpack after do_apt_fetch
-
-do_cleanall_apt[nostamp] = "1"
-do_cleanall_apt() {
- for uri in "${SRC_APT}"; do
- rm -rf "${DEBSRCDIR}/${BASE_DISTRO}-${BASE_DISTRO_CODENAME}/$uri"
- done
-}
-
def get_package_srcdir(d):
s = os.path.abspath(d.getVar("S"))
workdir = os.path.abspath(d.getVar("WORKDIR"))
diff --git a/meta/lib/aptsrc_fetcher.py b/meta/lib/aptsrc_fetcher.py
new file mode 100644
index 00000000..041f5445
--- /dev/null
+++ b/meta/lib/aptsrc_fetcher.py
@@ -0,0 +1,88 @@
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+from bb.fetch2 import FetchError
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
+
+class AptSrc(FetchMethod):
+ def supports(self, ud, d):
+ return ud.type in ['apt']
+
+ def urldata_init(self, ud, d):
+ ud.src_package = ud.url[len('apt://'):]
+ ud.host = ud.host.replace('=', '_')
+
+ base_distro = d.getVar('BASE_DISTRO')
+ codename = d.getVar('BASE_DISTRO_CODENAME')
+ ud.localfile='deb-src/' + base_distro + '-' + codename + '/' + ud.host
+
+ def download(self, ud, d):
+ bb.utils.exec_flat_python_func('isar_export_proxies', d)
+ bb.build.exec_func('schroot_create_configs', d)
+
+ sbuild_chroot = d.getVar('SBUILD_CHROOT')
+ session_id = runfetchcmd(f'schroot -q -b -c {sbuild_chroot}', d).strip()
+ logger.info(f'Started session: {session_id}')
+
+ repo_isar_dir = d.getVar('REPO_ISAR_DIR')
+ lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
+
+ try:
+ runfetchcmd(f'''
+ set -e
+ schroot -r -c {session_id} -d / -u root -- \
+ rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
+ schroot -r -c {session_id} -d / -- \
+ sh -c '
+ set -e
+ mkdir -p /downloads/{ud.localfile}
+ cd /downloads/{ud.localfile}
+ apt-get -y --download-only --only-source source {ud.src_package}
+ '
+ ''', d)
+ except (OSError, FetchError):
+ raise
+ finally:
+ bb.utils.unlockfile(lockfile)
+ runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
+ bb.build.exec_func('schroot_delete_configs', d)
+
+ def unpack(self, ud, rootdir, d):
+ bb.build.exec_func('schroot_create_configs', d)
+
+ sbuild_chroot = d.getVar('SBUILD_CHROOT')
+ session_id = runfetchcmd(f'schroot -q -b -c {sbuild_chroot}', d).strip()
+ logger.info(f'Started session: {session_id}')
+
+ pp = d.getVar('PP')
+ pps = d.getVar('PPS')
+ try:
+ runfetchcmd(f'''
+ set -e
+ schroot -r -c {session_id} -d / -u root -- \
+ rm /etc/apt/sources.list.d/isar-apt.list /etc/apt/preferences.d/isar-apt
+ schroot -r -c {session_id} -d / -- \
+ sh -c '
+ set -e
+ dscfile=$(apt-get -y -qq --print-uris --only-source source {ud.src_package} | \
+ cut -d " " -f2 | grep -E "\.dsc")
+ cp /downloads/{ud.localfile}/* {pp}
+ cd {pp}
+ mv -f {pps} {pps}.prev
+ dpkg-source -x "$dscfile" {pps}
+ find {pps}.prev -mindepth 1 -maxdepth 1 -exec mv {{}} {pps}/ \;
+ rmdir {pps}.prev
+ '
+ ''', d)
+ except (OSError, FetchError):
+ raise
+ finally:
+ runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
+ bb.build.exec_func('schroot_delete_configs', d)
+
+ def clean(self, ud, d):
+ bb.utils.remove(ud.localpath, recurse=True)
--
2.43.0
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/e20926f4-4da7-44b9-81d2-b1b19e707e80%40siemens.com.
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [PATCH v3] Convert apt source fetcher into native bitbake variant
2024-11-29 14:05 [PATCH v3] Convert apt source fetcher into native bitbake variant 'Jan Kiszka' via isar-users
@ 2024-12-06 7:30 ` Uladzimir Bely
0 siblings, 0 replies; 2+ messages in thread
From: Uladzimir Bely @ 2024-12-06 7:30 UTC (permalink / raw)
To: Jan Kiszka, isar-users
On Fri, 2024-11-29 at 22:05 +0800, Jan Kiszka wrote:
> From: Jan Kiszka <jan.kiszka@siemens.com>
>
> There is no major functional difference, but we no longer have to
> manipulate SRC_URI by registering an official fetcher for apt://.
>
> As the fetching no longer takes place in separate tasks, do_fetch and
> do_unpack need to gain the extra flags that were so far assigned to
> apt_fetch and apt_unpack. That happens conditionally, i.e. only if
> SRC_URI actually contains an apt URL.
>
> One difference to the original version is the possibility - even if
> practically of minor relevance - to unpack multiple apt sources into
> S.
> The old version contained a loop but was directing dpkg-source to a
> pre-existing dir which would have failed on the second iteration. The
> new version now folds the results together after each step.
>
> There is no progress report realized because dpkg-source
> unfortunately
> does not provide information upfront to make this predictable, thus
> expressible in form of percentage.
>
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
>
> Changes in v3:
> - keep split by distro codename in downloads/deb-src
>
> meta/classes/dpkg-base.bbclass | 104 ++++---------------------------
> --
> meta/lib/aptsrc_fetcher.py | 88 ++++++++++++++++++++++++++++
> 2 files changed, 99 insertions(+), 93 deletions(-)
> create mode 100644 meta/lib/aptsrc_fetcher.py
>
> diff --git a/meta/classes/dpkg-base.bbclass b/meta/classes/dpkg-
> base.bbclass
> index b4ea8e17..c02c07a8 100644
> --- a/meta/classes/dpkg-base.bbclass
> +++ b/meta/classes/dpkg-base.bbclass
> @@ -79,110 +79,28 @@ do_adjust_git[lockfiles] +=
> "${DL_DIR}/git/isar.lock"
> inherit patch
> addtask patch after do_adjust_git
>
> -SRC_APT ?= ""
> -
> -# filter out all "apt://" URIs out of SRC_URI and stick them into
> SRC_APT
> python() {
> - src_uri = (d.getVar('SRC_URI', False) or "").split()
> + from bb.fetch2 import methods
>
> - prefix = "apt://"
> - src_apt = []
> - for u in src_uri:
> - if u.startswith(prefix):
> - src_apt.append(u[len(prefix) :])
> - d.setVar('SRC_URI:remove', u)
> + # apt-src fetcher
> + import aptsrc_fetcher
> + methods.append(aptsrc_fetcher.AptSrc())
>
> - d.prependVar('SRC_APT', ' '.join(src_apt))
> + src_uri = (d.getVar('SRC_URI', False) or "").split()
> + for u in src_uri:
> + if u.startswith("apt://"):
> + d.appendVarFlag('do_fetch', 'depends',
> d.getVar('SCHROOT_DEP'))
>
> - if len(d.getVar('SRC_APT').strip()) > 0:
> - bb.build.addtask('apt_unpack', 'do_patch', '', d)
> - bb.build.addtask('cleanall_apt', 'do_cleanall', '', d)
> + d.appendVarFlag('do_unpack', 'cleandirs', d.getVar('S'))
> + d.setVarFlag('do_unpack', 'network',
> d.getVar('TASK_USE_SUDO'))
> + break
>
> # container docker fetcher
> import container_fetcher
> - from bb.fetch2 import methods
>
> methods.append(container_fetcher.Container())
> }
>
> -do_apt_fetch() {
> - E="${@ isar_export_proxies(d)}"
> - schroot_create_configs
> -
> - session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
> - echo "Started session: ${session_id}"
> -
> - schroot_cleanup() {
> - schroot -q -f -e -c ${session_id} > /dev/null 2>&1
> - schroot_delete_configs
> - }
> - trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> - trap 'schroot_cleanup' EXIT
> -
> - schroot -r -c ${session_id} -d / -u root -- \
> - rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> - schroot -r -c ${session_id} -d / -- \
> - sh -c '
> - set -e
> - for uri in $2; do
> - mkdir -p /downloads/deb-src/"$1"/${uri}
> - cd /downloads/deb-src/"$1"/${uri}
> - apt-get -y --download-only --only-source source
> ${uri}
> - done' \
> - my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> "${SRC_APT}"
> -
> - schroot -e -c ${session_id}
> - schroot_delete_configs
> -}
> -
> -addtask apt_fetch
> -do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
> -do_apt_fetch[network] = "${TASK_USE_NETWORK_AND_SUDO}"
> -
> -# Add dependency from the correct schroot: host or target
> -do_apt_fetch[depends] += "${SCHROOT_DEP}"
> -
> -do_apt_unpack() {
> - rm -rf ${S}
> - schroot_create_configs
> -
> - session_id=$(schroot -q -b -c ${SBUILD_CHROOT})
> - echo "Started session: ${session_id}"
> -
> - schroot_cleanup() {
> - schroot -q -f -e -c ${session_id} > /dev/null 2>&1
> - schroot_delete_configs
> - }
> - trap 'exit 1' INT HUP QUIT TERM ALRM USR1
> - trap 'schroot_cleanup' EXIT
> -
> - schroot -r -c ${session_id} -d / -u root -- \
> - rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> - schroot -r -c ${session_id} -d / -- \
> - sh -c '
> - set -e
> - for uri in $2; do
> - dscfile="$(apt-get -y -qq --print-uris --only-source
> source $uri | cut -d " " -f2 | grep -E "*.dsc")"
> - cd ${PP}
> - cp /downloads/deb-src/"${1}"/${uri}/* ${PP}
> - dpkg-source -x "${dscfile}" "${PPS}"
> - done' \
> - my_script "${BASE_DISTRO}-${BASE_DISTRO_CODENAME}"
> "${SRC_APT}"
> -
> - schroot -e -c ${session_id}
> - schroot_delete_configs
> -}
> -do_apt_unpack[network] = "${TASK_USE_SUDO}"
> -
> -addtask apt_unpack after do_apt_fetch
> -
> -do_cleanall_apt[nostamp] = "1"
> -do_cleanall_apt() {
> - for uri in "${SRC_APT}"; do
> - rm -rf "${DEBSRCDIR}/${BASE_DISTRO}-
> ${BASE_DISTRO_CODENAME}/$uri"
> - done
> -}
> -
> def get_package_srcdir(d):
> s = os.path.abspath(d.getVar("S"))
> workdir = os.path.abspath(d.getVar("WORKDIR"))
> diff --git a/meta/lib/aptsrc_fetcher.py b/meta/lib/aptsrc_fetcher.py
> new file mode 100644
> index 00000000..041f5445
> --- /dev/null
> +++ b/meta/lib/aptsrc_fetcher.py
> @@ -0,0 +1,88 @@
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2024
> +#
> +# SPDX-License-Identifier: MIT
> +
> +from bb.fetch2 import FetchError
> +from bb.fetch2 import FetchMethod
> +from bb.fetch2 import logger
> +from bb.fetch2 import runfetchcmd
> +
> +class AptSrc(FetchMethod):
> + def supports(self, ud, d):
> + return ud.type in ['apt']
> +
> + def urldata_init(self, ud, d):
> + ud.src_package = ud.url[len('apt://'):]
> + ud.host = ud.host.replace('=', '_')
> +
> + base_distro = d.getVar('BASE_DISTRO')
> + codename = d.getVar('BASE_DISTRO_CODENAME')
> + ud.localfile='deb-src/' + base_distro + '-' + codename + '/'
> + ud.host
> +
> + def download(self, ud, d):
> + bb.utils.exec_flat_python_func('isar_export_proxies', d)
> + bb.build.exec_func('schroot_create_configs', d)
> +
> + sbuild_chroot = d.getVar('SBUILD_CHROOT')
> + session_id = runfetchcmd(f'schroot -q -b -c
> {sbuild_chroot}', d).strip()
> + logger.info(f'Started session: {session_id}')
> +
> + repo_isar_dir = d.getVar('REPO_ISAR_DIR')
> + lockfile = bb.utils.lockfile(f'{repo_isar_dir}/isar.lock')
> +
> + try:
> + runfetchcmd(f'''
> + set -e
> + schroot -r -c {session_id} -d / -u root -- \
> + rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> + schroot -r -c {session_id} -d / -- \
> + sh -c '
> + set -e
> + mkdir -p /downloads/{ud.localfile}
> + cd /downloads/{ud.localfile}
> + apt-get -y --download-only --only-source
> source {ud.src_package}
> + '
> + ''', d)
> + except (OSError, FetchError):
> + raise
> + finally:
> + bb.utils.unlockfile(lockfile)
> + runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
> + bb.build.exec_func('schroot_delete_configs', d)
> +
> + def unpack(self, ud, rootdir, d):
> + bb.build.exec_func('schroot_create_configs', d)
> +
> + sbuild_chroot = d.getVar('SBUILD_CHROOT')
> + session_id = runfetchcmd(f'schroot -q -b -c
> {sbuild_chroot}', d).strip()
> + logger.info(f'Started session: {session_id}')
> +
> + pp = d.getVar('PP')
> + pps = d.getVar('PPS')
> + try:
> + runfetchcmd(f'''
> + set -e
> + schroot -r -c {session_id} -d / -u root -- \
> + rm /etc/apt/sources.list.d/isar-apt.list
> /etc/apt/preferences.d/isar-apt
> + schroot -r -c {session_id} -d / -- \
> + sh -c '
> + set -e
> + dscfile=$(apt-get -y -qq --print-uris --
> only-source source {ud.src_package} | \
> + cut -d " " -f2 | grep -E "\.dsc")
> + cp /downloads/{ud.localfile}/* {pp}
> + cd {pp}
> + mv -f {pps} {pps}.prev
> + dpkg-source -x "$dscfile" {pps}
> + find {pps}.prev -mindepth 1 -maxdepth 1 -
> exec mv {{}} {pps}/ \;
> + rmdir {pps}.prev
> + '
> + ''', d)
> + except (OSError, FetchError):
> + raise
> + finally:
> + runfetchcmd(f'schroot -q -f -e -c {session_id}', d)
> + bb.build.exec_func('schroot_delete_configs', d)
> +
> + def clean(self, ud, d):
> + bb.utils.remove(ud.localpath, recurse=True)
Applied to next, thanks.
--
Best regards,
Uladzimir.
--
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/0ff493b508642989c5239247f9027fbcfe233cc6.camel%40ilbers.de.
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2024-12-06 7:30 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-11-29 14:05 [PATCH v3] Convert apt source fetcher into native bitbake variant 'Jan Kiszka' via isar-users
2024-12-06 7:30 ` Uladzimir Bely
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox