From mboxrd@z Thu Jan 1 00:00:00 1970 X-GM-THRID: 7192672744921628672 X-Received: by 2002:a1f:7c82:0:b0:3da:fa83:5297 with SMTP id x124-20020a1f7c82000000b003dafa835297mr4279023vkc.6.1674674627538; Wed, 25 Jan 2023 11:23:47 -0800 (PST) X-BeenThere: isar-users@googlegroups.com Received: by 2002:a1f:9d97:0:b0:3e1:cb2e:e15d with SMTP id g145-20020a1f9d97000000b003e1cb2ee15dls3529314vke.1.-pod-prod-gmail; Wed, 25 Jan 2023 11:23:46 -0800 (PST) X-Google-Smtp-Source: AMrXdXu2h1mZXH6aFLZ0ZWhJp2gm8GXg3mScjwx8eQGs4Sj6YyO/tQqLj9ls1rPuUeaUnXqPKpMJ X-Received: by 2002:a05:6122:48e:b0:3d5:5055:f22 with SMTP id o14-20020a056122048e00b003d550550f22mr18487376vkn.1.1674674626325; Wed, 25 Jan 2023 11:23:46 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1674674626; cv=none; d=google.com; s=arc-20160816; b=tPApInT7OhaMZ0h9uNWcSlfRe6zLuUOga/HkDAQHEsXM0OCA/DZuAF/pqNrXVSt5dS FJsr5DnXpbQ2UkzuC4XOZPSNyagrvNZBrUlHZpwMRlZtFD0Hd39QJ+lLRiAro7fL+i9R 5S6/QUt32AzFZNdVgrsDH8ukl+Oti2j7I2lN9QDzGkB351EsMeUS7mZar0Vuu+lSJIYM XVuRyJZ130JOHd937jVOTwZvLNIoitXrBsjDsVZdAP1FWbfQ+qJHP9C7JWro9qqF5AaG SAlKzU1AJFL7x5mQ4WQFxkY0LnSzxC3767rvai5+vQCRwQtoK+bqaXWbwCbJ46Dbe9w7 PKeg== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from; bh=BDlXvEVC753+d3VKZKminbZ34js61ENq3x7xr8ISCUQ=; b=WN4ryfaarZcTPMGKCltqXnwsogcBEfNKU85I23y/TNnuCJzHvpcLoUlO4Sog0rkLyg 89XzeOpKZ2TgeUcpfwuqhWNrRuC1U4NAGPakuCY0qbxmnLTUp2IlH7FhxMMAJMm4AvRr dZbtLsHQ/AlH7giZza8Xmy1zAl9ve4OIkkp9O8e0GCzoxCiNqfBwV+Dgt8OPxbIHBpsK itAwJ75ZRH+Qn7xEbxMUJuLOSHSqstYcO/z7xPSO4Br1QLyQMSbTNks4GoT17D6YQvMi OgN9Gtwj2I5wz0e6Qo7Jz4K8zKeBSiGYgxbGDoAsOAZ7uwIW4rE17Y8JfrDBK0nyCtpm Z2cw== ARC-Authentication-Results: i=1; gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Return-Path: Received: from shymkent.ilbers.de (shymkent.ilbers.de. [85.214.156.166]) by gmr-mx.google.com with ESMTPS id l22-20020a05612210d600b003e4251d7a1fsi271599vko.3.2023.01.25.11.23.45 for (version=TLS1_2 cipher=ECDHE-ECDSA-CHACHA20-POLY1305 bits=256/256); Wed, 25 Jan 2023 11:23:46 -0800 (PST) Received-SPF: pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) client-ip=85.214.156.166; Authentication-Results: gmr-mx.google.com; spf=pass (google.com: domain of amikan@ilbers.de designates 85.214.156.166 as permitted sender) smtp.mailfrom=amikan@ilbers.de Received: from user-B660.. (IN-213-226-141-182.bitemobile.lv [213.226.141.182] (may be forged)) (authenticated bits=0) by shymkent.ilbers.de (8.15.2/8.15.2/Debian-8+deb9u1) with ESMTPSA id 30PJNdaH028378 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Wed, 25 Jan 2023 20:23:43 +0100 From: Anton Mikanovich To: isar-users@googlegroups.com Cc: Anton Mikanovich Subject: [PATCH v8 02/20] scripts/contrib: add override conversion script Date: Wed, 25 Jan 2023 21:23:19 +0200 Message-Id: <20230125192337.86869-3-amikan@ilbers.de> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230125192337.86869-1-amikan@ilbers.de> References: <20230125192337.86869-1-amikan@ilbers.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.0 required=5.0 tests=ALL_TRUSTED autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shymkent.ilbers.de X-TUID: bh5ihRTsUj/d This adds a script developed to migrate metadata to use the new override syntax. It is a bit rough but since its for a single use with validation, it doesn't need to be perfect. It is run simply as: scripts/contrib/convert-overrides.py For other layers it would need updating for override names and exclusions for functions/variable names with "append", "prepend" or "remove" in them. (From Yocto kirkstone-4.0.5 rev: fbdf93f43ff4b876487e1f26752598ec8abcb46e) Signed-off-by: Anton Mikanovich --- scripts/contrib/convert-overrides.py | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 scripts/contrib/convert-overrides.py diff --git a/scripts/contrib/convert-overrides.py b/scripts/contrib/convert-overrides.py new file mode 100755 index 0000000..4d41a4c --- /dev/null +++ b/scripts/contrib/convert-overrides.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# +# Conversion script to add new override syntax to existing bitbake metadata +# +# Copyright (C) 2021 Richard Purdie +# +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# To use this script on a new layer you need to list the overrides the +# layer is known to use in the list below. +# +# Known constraint: Matching is 'loose' and in particular will find variable +# and function names with "_append" and "_remove" in them. Those need to be +# filtered out manually or in the skip list below. +# + +import re +import os +import sys +import tempfile +import shutil +import mimetypes + +if len(sys.argv) < 2: + print("Please specify a directory to run the conversion script against.") + sys.exit(1) + +# List of strings to treat as overrides +vars = ["append", "prepend", "remove"] +vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"] +vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"] +vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"] +vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"] +vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"] +vars = vars + ["tune-", "pn-", "forcevariable"] +vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"] +vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"] +vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"] +vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"] +vars = vars + ["linux-gnueabi", "eabi"] +vars = vars + ["virtclass-multilib", "virtclass-mcextend"] + +# List of strings to treat as overrides but only with whitespace following or another override (more restricted matching). +# Handles issues with arc matching arch. +shortvars = ["arc", "mips", "mipsel", "sh4"] + +# Variables which take packagenames as an override +packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY", + "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE", + "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst", + "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA", + "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] + +# Expressions to skip if encountered, these are not overrides +skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"] +skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"] +skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"] +skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"] +skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"] +skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"] +skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"] +skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"] +skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"] + +imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] +packagevars = packagevars + imagevars + +vars_re = {} +for exp in vars: + vars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp), r"\1:" + exp) + +shortvars_re = {} +for exp in shortvars: + shortvars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp + '([\(\'"\s:])'), r"\1:" + exp + r"\3") + +package_re = {} +for exp in packagevars: + package_re[exp] = (re.compile('(^|[#\'"\s\-\+]+)' + exp + '_' + '([$a-z"\'\s%\[<{\\\*].)'), r"\1" + exp + r":\2") + +# Other substitutions to make +subs = { + 'r = re.compile("([^:]+):\s*(.*)")' : 'r = re.compile("(^.+?):\s+(.*)")', + "val = d.getVar('%s_%s' % (var, pkg))" : "val = d.getVar('%s:%s' % (var, pkg))", + "f.write('%s_%s: %s\\n' % (var, pkg, encode(val)))" : "f.write('%s:%s: %s\\n' % (var, pkg, encode(val)))", + "d.getVar('%s_%s' % (scriptlet_name, pkg))" : "d.getVar('%s:%s' % (scriptlet_name, pkg))", + 'ret.append(v + "_" + p)' : 'ret.append(v + ":" + p)', +} + +def processfile(fn): + print("processing file '%s'" % fn) + try: + fh, abs_path = tempfile.mkstemp() + with os.fdopen(fh, 'w') as new_file: + with open(fn, "r") as old_file: + for line in old_file: + skip = False + for s in skips: + if s in line: + skip = True + if "ptest_append" in line or "ptest_remove" in line or "ptest_prepend" in line: + skip = False + for sub in subs: + if sub in line: + line = line.replace(sub, subs[sub]) + skip = True + if not skip: + for pvar in packagevars: + line = package_re[pvar][0].sub(package_re[pvar][1], line) + for var in vars: + line = vars_re[var][0].sub(vars_re[var][1], line) + for shortvar in shortvars: + line = shortvars_re[shortvar][0].sub(shortvars_re[shortvar][1], line) + if "pkg_postinst:ontarget" in line: + line = line.replace("pkg_postinst:ontarget", "pkg_postinst_ontarget") + new_file.write(line) + shutil.copymode(fn, abs_path) + os.remove(fn) + shutil.move(abs_path, fn) + except UnicodeDecodeError: + pass + +ourname = os.path.basename(sys.argv[0]) +ourversion = "0.9.3" + +if os.path.isfile(sys.argv[1]): + processfile(sys.argv[1]) + sys.exit(0) + +for targetdir in sys.argv[1:]: + print("processing directory '%s'" % targetdir) + for root, dirs, files in os.walk(targetdir): + for name in files: + if name == ourname: + continue + fn = os.path.join(root, name) + if os.path.islink(fn): + continue + if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"): + continue + processfile(fn) + +print("All files processed with version %s" % ourversion) -- 2.34.1