Re: [PATCH v2 2/2] bitbake: Remove custom exception backtrace formatting

["'Jan Kiszka' via isar-users" <isar-users@googlegroups.com>]

Missing original author and signed-off.

Jan

On 06.03.26 17:15, Felix Moessbauer wrote:
> Backport of upstream bitbake patch c25e7ed128b9fd5b53d28d678238e2f3af52ef8b.
> 
> Removes the code in bitbake to show custom backtrace formatting for
> exceptions. In particular, the bitbake exception code prints function
> arguments, which while helpful is a security problem when passwords and
> other secrets can be passed as function arguments.
> 
> As it turns out, the handling of the custom serialized exception stack
> frames was pretty much made obsolete by d7db75020ed ("event/msg: Pass
> formatted exceptions"), which changed the events to pass a preformatted
> stacktrack list of strings, but the passing of the serialized data was
> never removed.
> 
> Change all the code to use the python traceback API to format exceptions
> instead of the custom code; conveniently traceback.format_exception()
> also returns a list of stack trace strings, so it can be used as a drop
> in replacement for bb.exception.format_exception()
> 
> [Felix: adjusted to bitbake path in isar repo]
> 
> Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com>
> ---
>  bitbake/lib/bb/cooker.py      | 32 +++++++++---
>  bitbake/lib/bb/event.py       |  9 +---
>  bitbake/lib/bb/exceptions.py  | 96 -----------------------------------
>  bitbake/lib/bb/msg.py         |  4 --
>  bitbake/lib/bb/ui/teamcity.py |  5 --
>  5 files changed, 25 insertions(+), 121 deletions(-)
>  delete mode 100644 bitbake/lib/bb/exceptions.py
> 
> diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
> index c5bfef55..701cf51b 100644
> --- a/bitbake/lib/bb/cooker.py
> +++ b/bitbake/lib/bb/cooker.py
> @@ -17,7 +17,7 @@ import threading
>  from io import StringIO, UnsupportedOperation
>  from contextlib import closing
>  from collections import defaultdict, namedtuple
> -import bb, bb.exceptions, bb.command
> +import bb, bb.command
>  from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
>  import queue
>  import signal
> @@ -2098,7 +2098,6 @@ class Parser(multiprocessing.Process):
>          except Exception as exc:
>              tb = sys.exc_info()[2]
>              exc.recipe = filename
> -            exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
>              return True, None, exc
>          # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
>          # and for example a worker thread doesn't just exit on its own in response to
> @@ -2299,8 +2298,12 @@ class CookerParser(object):
>              return False
>          except ParsingFailure as exc:
>              self.error += 1
> -            logger.error('Unable to parse %s: %s' %
> -                     (exc.recipe, bb.exceptions.to_string(exc.realexception)))
> +
> +            exc_desc = str(exc)
> +            if isinstance(exc, SystemExit) and not isinstance(exc.code, str):
> +                exc_desc = 'Exited with "%d"' % exc.code
> +
> +            logger.error('Unable to parse %s: %s' % (exc.recipe, exc_desc))
>              self.shutdown(clean=False)
>              return False
>          except bb.parse.ParseError as exc:
> @@ -2309,20 +2312,33 @@ class CookerParser(object):
>              self.shutdown(clean=False, eventmsg=str(exc))
>              return False
>          except bb.data_smart.ExpansionError as exc:
> +            def skip_frames(f, fn_prefix):
> +                while f and f.tb_frame.f_code.co_filename.startswith(fn_prefix):
> +                    f = f.tb_next
> +                return f
> +
>              self.error += 1
>              bbdir = os.path.dirname(__file__) + os.sep
> -            etype, value, _ = sys.exc_info()
> -            tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
> +            etype, value, tb = sys.exc_info()
> +
> +            # Remove any frames where the code comes from bitbake. This
> +            # prevents deep (and pretty useless) backtraces for expansion error
> +            tb = skip_frames(tb, bbdir)
> +            cur = tb
> +            while cur:
> +                cur.tb_next = skip_frames(cur.tb_next, bbdir)
> +                cur = cur.tb_next
> +
>              logger.error('ExpansionError during parsing %s', value.recipe,
>                           exc_info=(etype, value, tb))
>              self.shutdown(clean=False)
>              return False
>          except Exception as exc:
>              self.error += 1
> -            etype, value, tb = sys.exc_info()
> +            _, value, _ = sys.exc_info()
>              if hasattr(value, "recipe"):
>                  logger.error('Unable to parse %s' % value.recipe,
> -                            exc_info=(etype, value, exc.traceback))
> +                            exc_info=sys.exc_info())
>              else:
>                  # Most likely, an exception occurred during raising an exception
>                  import traceback
> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> index 4761c868..952c85c0 100644
> --- a/bitbake/lib/bb/event.py
> +++ b/bitbake/lib/bb/event.py
> @@ -19,7 +19,6 @@ import sys
>  import threading
>  import traceback
>  
> -import bb.exceptions
>  import bb.utils
>  
>  # This is the pid for which we should generate the event. This is set when
> @@ -759,13 +758,7 @@ class LogHandler(logging.Handler):
>  
>      def emit(self, record):
>          if record.exc_info:
> -            etype, value, tb = record.exc_info
> -            if hasattr(tb, 'tb_next'):
> -                tb = list(bb.exceptions.extract_traceback(tb, context=3))
> -            # Need to turn the value into something the logging system can pickle
> -            record.bb_exc_info = (etype, value, tb)
> -            record.bb_exc_formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
> -            value = str(value)
> +            record.bb_exc_formatted = traceback.format_exception(*record.exc_info)
>              record.exc_info = None
>          fire(record, None)
>  
> diff --git a/bitbake/lib/bb/exceptions.py b/bitbake/lib/bb/exceptions.py
> deleted file mode 100644
> index 801db9c8..00000000
> --- a/bitbake/lib/bb/exceptions.py
> +++ /dev/null
> @@ -1,96 +0,0 @@
> -#
> -# Copyright BitBake Contributors
> -#
> -# SPDX-License-Identifier: GPL-2.0-only
> -#
> -
> -import inspect
> -import traceback
> -import bb.namedtuple_with_abc
> -from collections import namedtuple
> -
> -
> -class TracebackEntry(namedtuple.abc):
> -    """Pickleable representation of a traceback entry"""
> -    _fields = 'filename lineno function args code_context index'
> -    _header = '  File "{0.filename}", line {0.lineno}, in {0.function}{0.args}'
> -
> -    def format(self, formatter=None):
> -        if not self.code_context:
> -            return self._header.format(self) + '\n'
> -
> -        formatted = [self._header.format(self) + ':\n']
> -
> -        for lineindex, line in enumerate(self.code_context):
> -            if formatter:
> -                line = formatter(line)
> -
> -            if lineindex == self.index:
> -                formatted.append('    >%s' % line)
> -            else:
> -                formatted.append('     %s' % line)
> -        return formatted
> -
> -    def __str__(self):
> -        return ''.join(self.format())
> -
> -def _get_frame_args(frame):
> -    """Get the formatted arguments and class (if available) for a frame"""
> -    arginfo = inspect.getargvalues(frame)
> -
> -    try:
> -        if not arginfo.args:
> -            return '', None
> -    # There have been reports from the field of python 2.6 which doesn't 
> -    # return a namedtuple here but simply a tuple so fallback gracefully if
> -    # args isn't present.
> -    except AttributeError:
> -        return '', None
> -
> -    firstarg = arginfo.args[0]
> -    if firstarg == 'self':
> -        self = arginfo.locals['self']
> -        cls = self.__class__.__name__
> -
> -        arginfo.args.pop(0)
> -        del arginfo.locals['self']
> -    else:
> -        cls = None
> -
> -    formatted = inspect.formatargvalues(*arginfo)
> -    return formatted, cls
> -
> -def extract_traceback(tb, context=1):
> -    frames = inspect.getinnerframes(tb, context)
> -    for frame, filename, lineno, function, code_context, index in frames:
> -        formatted_args, cls = _get_frame_args(frame)
> -        if cls:
> -            function = '%s.%s' % (cls, function)
> -        yield TracebackEntry(filename, lineno, function, formatted_args,
> -                             code_context, index)
> -
> -def format_extracted(extracted, formatter=None, limit=None):
> -    if limit:
> -        extracted = extracted[-limit:]
> -
> -    formatted = []
> -    for tracebackinfo in extracted:
> -        formatted.extend(tracebackinfo.format(formatter))
> -    return formatted
> -
> -
> -def format_exception(etype, value, tb, context=1, limit=None, formatter=None):
> -    formatted = ['Traceback (most recent call last):\n']
> -
> -    if hasattr(tb, 'tb_next'):
> -        tb = extract_traceback(tb, context)
> -
> -    formatted.extend(format_extracted(tb, formatter, limit))
> -    formatted.extend(traceback.format_exception_only(etype, value))
> -    return formatted
> -
> -def to_string(exc):
> -    if isinstance(exc, SystemExit):
> -        if not isinstance(exc.code, str):
> -            return 'Exited with "%d"' % exc.code
> -    return str(exc)
> diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
> index 3e18596f..4f616ff4 100644
> --- a/bitbake/lib/bb/msg.py
> +++ b/bitbake/lib/bb/msg.py
> @@ -89,10 +89,6 @@ class BBLogFormatter(logging.Formatter):
>              msg = logging.Formatter.format(self, record)
>          if hasattr(record, 'bb_exc_formatted'):
>              msg += '\n' + ''.join(record.bb_exc_formatted)
> -        elif hasattr(record, 'bb_exc_info'):
> -            etype, value, tb = record.bb_exc_info
> -            formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
> -            msg += '\n' + ''.join(formatted)
>          return msg
>  
>      def colorize(self, record):
> diff --git a/bitbake/lib/bb/ui/teamcity.py b/bitbake/lib/bb/ui/teamcity.py
> index fca46c28..7eeaab8d 100644
> --- a/bitbake/lib/bb/ui/teamcity.py
> +++ b/bitbake/lib/bb/ui/teamcity.py
> @@ -30,7 +30,6 @@ import bb.build
>  import bb.command
>  import bb.cooker
>  import bb.event
> -import bb.exceptions
>  import bb.runqueue
>  from bb.ui import uihelper
>  
> @@ -102,10 +101,6 @@ class TeamcityLogFormatter(logging.Formatter):
>          details = ""
>          if hasattr(record, 'bb_exc_formatted'):
>              details = ''.join(record.bb_exc_formatted)
> -        elif hasattr(record, 'bb_exc_info'):
> -            etype, value, tb = record.bb_exc_info
> -            formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
> -            details = ''.join(formatted)
>  
>          if record.levelno in [bb.msg.BBLogFormatter.ERROR, bb.msg.BBLogFormatter.CRITICAL]:
>              # ERROR gets a separate errorDetails field

-- 
Siemens AG, Foundational Technologies
Linux Expert Center

-- 
You received this message because you are subscribed to the Google Groups "isar-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to isar-users+unsubscribe@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/isar-users/c22c8903-caf2-415f-8d61-e5f2003290ec%40siemens.com.