Re: [PATCH v2] rootfs: Improve progress reporting

[Uladzimir Bely <ubely@ilbers.de>]

On Sat, 2024-08-24 at 09:24 +0200, 'Jan Kiszka' via isar-users wrote:
> From: Jan Kiszka <jan.kiszka@siemens.com>
> 
> Add a custom progress reporter that visualizes the state of the
> rootfs_install_pkgs_download and rootfs_install_pkgs_install
> functions
> by parsing dpkg output. This is achieved by reading the number of
> upgraded and new installed packages first and then distributing the
> effort equally over each package.
> 
> There is another specialty: As do_rootfs_install already created a
> MultiStageProgressReporter, we forward it via the metadata object and
> report from functions as stage progress so that the bar moves forward
> consistently.
> 
> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
> ---
> 
> Changes in v2:
>  - catch all "Setting up"
>  - scale down to 99% max to account for that we already progress
> before
>    a package step is done
> 
>  meta/classes/rootfs.bbclass |  9 +++++
>  meta/lib/rootfs_progress.py | 67
> +++++++++++++++++++++++++++++++++++++
>  2 files changed, 76 insertions(+)
>  create mode 100644 meta/lib/rootfs_progress.py
> 
> diff --git a/meta/classes/rootfs.bbclass
> b/meta/classes/rootfs.bbclass
> index fdb25eaa..f0abd795 100644
> --- a/meta/classes/rootfs.bbclass
> +++ b/meta/classes/rootfs.bbclass
> @@ -76,6 +76,12 @@ rootfs_do_qemu() {
>  BOOTSTRAP_SRC = "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-
> host_${DISTRO}-${DISTRO_ARCH}"
>  BOOTSTRAP_SRC:${ROOTFS_ARCH} =
> "${DEPLOY_DIR_BOOTSTRAP}/${ROOTFS_DISTRO}-${ROOTFS_ARCH}"
>  
> +def rootfs_extra_import(d):
> +    bb.utils._context["rootfs_progress"] =
> __import__("rootfs_progress")
> +    return ""
> +
> +ROOTFS_EXTRA_IMPORTED := "${@rootfs_extra_import(d)}"
> +
>  rootfs_prepare[weight] = "25"
>  rootfs_prepare(){
>      sudo cp -Trpfx --reflink=auto '${BOOTSTRAP_SRC}/' '${ROOTFSDIR}'
> @@ -153,6 +159,7 @@ rootfs_import_package_cache() {
>  
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_download"
>  rootfs_install_pkgs_download[weight] = "600"
> +rootfs_install_pkgs_download[progress] =
> "custom:rootfs_progress.PkgsDownloadProgressHandler"
>  rootfs_install_pkgs_download[isar-apt-lock] = "release-after"
>  rootfs_install_pkgs_download[network] =
> "${TASK_USE_NETWORK_AND_SUDO}"
>  rootfs_install_pkgs_download() {
> @@ -178,6 +185,7 @@ rootfs_install_clean_files() {
>  
>  ROOTFS_INSTALL_COMMAND += "rootfs_install_pkgs_install"
>  rootfs_install_pkgs_install[weight] = "8000"
> +rootfs_install_pkgs_install[progress] =
> "custom:rootfs_progress.PkgsInstallProgressHandler"
>  rootfs_install_pkgs_install[network] = "${TASK_USE_SUDO}"
>  rootfs_install_pkgs_install() {
>      sudo -E chroot "${ROOTFSDIR}" \
> @@ -206,6 +214,7 @@ python do_rootfs_install() {
>                       for i in cmds]
>  
>      progress_reporter = bb.progress.MultiStageProgressReporter(d,
> stage_weights)
> +    d.rootfs_progress = progress_reporter
>  
>      for cmd in cmds:
>          progress_reporter.next_stage()
> diff --git a/meta/lib/rootfs_progress.py
> b/meta/lib/rootfs_progress.py
> new file mode 100644
> index 00000000..f808852f
> --- /dev/null
> +++ b/meta/lib/rootfs_progress.py
> @@ -0,0 +1,67 @@
> +# This software is a part of ISAR.
> +# Copyright (c) Siemens AG, 2024
> +#
> +# SPDX-License-Identifier: MIT
> +
> +import bb.progress
> +import re
> +
> +
> +class PkgsProgressHandler(bb.progress.ProgressHandler):
> +    def __init__(self, d, outfile):
> +        self._outfile = outfile
> +        self._progress = d.rootfs_progress
> +        self._progress.update(0)
> +        self._linebuffer = ''
> +        self._num_pkgs = 0
> +        self._pkg = 0
> +        self._stage = 'prepare'
> +
> +    def write(self, string):
> +        self._outfile.write(string)
> +        self._linebuffer += string
> +        while True:
> +            breakpos = self._linebuffer.find('\n') + 1
> +            if breakpos == 0:
> +                break
> +            line = self._linebuffer[:breakpos]
> +            self._linebuffer = self._linebuffer[breakpos:]
> +
> +            if self._stage == 'prepare':
> +                match = re.search(
> +                    r'^([0-9]+) upgraded, ([0-9]+) newly installed',
> line)
> +                if match:
> +                    self._num_pkgs = int(match.group(1)) +
> int(match.group(2))
> +                    self._stage = 'post-prepare'
> +            else:
> +                self.process_line(line)
> +
> +    def process_line(self, line):
> +        return
> +
> +
> +class PkgsDownloadProgressHandler(PkgsProgressHandler):
> +    def __init__(self, d, outfile, otherargs=None):
> +        super().__init__(d, outfile)
> +
> +    def process_line(self, line):
> +        if line.startswith('Get:'):
> +            self._pkg += 1
> +            self._progress.update(99 * self._pkg / self._num_pkgs)
> +
> +
> +class PkgsInstallProgressHandler(PkgsProgressHandler):
> +    def __init__(self, d, outfile, otherargs=None):
> +        self._pkg_set_up = 0
> +        super().__init__(d, outfile)
> +
> +    def process_line(self, line):
> +        if line.startswith('Preparing to unpack'):
> +            self._pkg += 1
> +        elif line.startswith('Setting up'):
> +            self._pkg_set_up += 1
> +        else:
> +            return
> +
> +        progress = 99 * (self._pkg + self._pkg_set_up) /
> (self._num_pkgs * 2)
> +        self._progress.update(progress)
> -- 
> 2.43.0
> 

Applied to next, thanks.

-- 
Best regards,
Uladzimir.

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/isar-users/a2db0f4f600e129ff72674662ae2291d5bf8d452.camel%40ilbers.de.