| #!/usr/bin/env python3 |
| # pylint: disable=R0902,R0912,R0914,R0915,R1702 |
| # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. |
| # SPDX-License-Identifier: GPL-2.0 |
| |
| """ |
| Parse ABI documentation and produce results from it. |
| """ |
| |
| import os |
| import re |
| import sys |
| |
| from concurrent import futures |
| from datetime import datetime |
| from random import shuffle |
| |
| from helpers import AbiDebug |
| |
| class SystemSymbols: |
| """Stores arguments for the class and initialize class vars""" |
| |
| def graph_add_file(self, path, link=None): |
| """ |
| add a file path to the sysfs graph stored at self.root |
| """ |
| |
| if path in self.files: |
| return |
| |
| name = "" |
| ref = self.root |
| for edge in path.split("/"): |
| name += edge + "/" |
| if edge not in ref: |
| ref[edge] = {"__name": [name.rstrip("/")]} |
| |
| ref = ref[edge] |
| |
| if link and link not in ref["__name"]: |
| ref["__name"].append(link.rstrip("/")) |
| |
| self.files.add(path) |
| |
| def print_graph(self, root_prefix="", root=None, level=0): |
| """Prints a reference tree graph using UTF-8 characters""" |
| |
| if not root: |
| root = self.root |
| level = 0 |
| |
| # Prevent endless traverse |
| if level > 5: |
| return |
| |
| if level > 0: |
| prefix = "├──" |
| last_prefix = "└──" |
| else: |
| prefix = "" |
| last_prefix = "" |
| |
| items = list(root.items()) |
| |
| names = root.get("__name", []) |
| for k, edge in items: |
| if k == "__name": |
| continue |
| |
| if not k: |
| k = "/" |
| |
| if len(names) > 1: |
| k += " links: " + ",".join(names[1:]) |
| |
| if edge == items[-1][1]: |
| print(root_prefix + last_prefix + k) |
| p = root_prefix |
| if level > 0: |
| p += " " |
| self.print_graph(p, edge, level + 1) |
| else: |
| print(root_prefix + prefix + k) |
| p = root_prefix + "│ " |
| self.print_graph(p, edge, level + 1) |
| |
| def _walk(self, root): |
| """ |
| Walk through sysfs to get all devnodes that aren't ignored. |
| |
| By default, uses /sys as sysfs mounting point. If another |
| directory is used, it replaces them to /sys at the patches. |
| """ |
| |
| with os.scandir(root) as obj: |
| for entry in obj: |
| path = os.path.join(root, entry.name) |
| if self.sysfs: |
| p = path.replace(self.sysfs, "/sys", count=1) |
| else: |
| p = path |
| |
| if self.re_ignore.search(p): |
| return |
| |
| # Handle link first to avoid directory recursion |
| if entry.is_symlink(): |
| real = os.path.realpath(path) |
| if not self.sysfs: |
| self.aliases[path] = real |
| else: |
| real = real.replace(self.sysfs, "/sys", count=1) |
| |
| # Add absfile location to graph if it doesn't exist |
| if not self.re_ignore.search(real): |
| # Add link to the graph |
| self.graph_add_file(real, p) |
| |
| elif entry.is_file(): |
| self.graph_add_file(p) |
| |
| elif entry.is_dir(): |
| self._walk(path) |
| |
| def __init__(self, abi, sysfs="/sys", hints=False): |
| """ |
| Initialize internal variables and get a list of all files inside |
| sysfs that can currently be parsed. |
| |
| Please notice that there are several entries on sysfs that aren't |
| documented as ABI. Ignore those. |
| |
| The real paths will be stored under self.files. Aliases will be |
| stored in separate, as self.aliases. |
| """ |
| |
| self.abi = abi |
| self.log = abi.log |
| |
| if sysfs != "/sys": |
| self.sysfs = sysfs.rstrip("/") |
| else: |
| self.sysfs = None |
| |
| self.hints = hints |
| |
| self.root = {} |
| self.aliases = {} |
| self.files = set() |
| |
| dont_walk = [ |
| # Those require root access and aren't documented at ABI |
| f"^{sysfs}/kernel/debug", |
| f"^{sysfs}/kernel/tracing", |
| f"^{sysfs}/fs/pstore", |
| f"^{sysfs}/fs/bpf", |
| f"^{sysfs}/fs/fuse", |
| |
| # This is not documented at ABI |
| f"^{sysfs}/module", |
| |
| f"^{sysfs}/fs/cgroup", # this is big and has zero docs under ABI |
| f"^{sysfs}/firmware", # documented elsewhere: ACPI, DT bindings |
| "sections|notes", # aren't actually part of ABI |
| |
| # kernel-parameters.txt - not easy to parse |
| "parameters", |
| ] |
| |
| self.re_ignore = re.compile("|".join(dont_walk)) |
| |
| print(f"Reading {sysfs} directory contents...", file=sys.stderr) |
| self._walk(sysfs) |
| |
| def check_file(self, refs, found): |
| """Check missing ABI symbols for a given sysfs file""" |
| |
| res_list = [] |
| |
| try: |
| for names in refs: |
| fname = names[0] |
| |
| res = { |
| "found": False, |
| "fname": fname, |
| "msg": "", |
| } |
| res_list.append(res) |
| |
| re_what = self.abi.get_regexes(fname) |
| if not re_what: |
| self.abi.log.warning(f"missing rules for {fname}") |
| continue |
| |
| for name in names: |
| for r in re_what: |
| if self.abi.debug & AbiDebug.UNDEFINED: |
| self.log.debug("check if %s matches '%s'", name, r.pattern) |
| if r.match(name): |
| res["found"] = True |
| if found: |
| res["msg"] += f" {fname}: regex:\n\t" |
| continue |
| |
| if self.hints and not res["found"]: |
| res["msg"] += f" {fname} not found. Tested regexes:\n" |
| for r in re_what: |
| res["msg"] += " " + r.pattern + "\n" |
| |
| except KeyboardInterrupt: |
| pass |
| |
| return res_list |
| |
| def _ref_interactor(self, root): |
| """Recursive function to interact over the sysfs tree""" |
| |
| for k, v in root.items(): |
| if isinstance(v, dict): |
| yield from self._ref_interactor(v) |
| |
| if root == self.root or k == "__name": |
| continue |
| |
| if self.abi.re_string: |
| fname = v["__name"][0] |
| if self.abi.re_string.search(fname): |
| yield v |
| else: |
| yield v |
| |
| |
| def get_fileref(self, all_refs, chunk_size): |
| """Interactor to group refs into chunks""" |
| |
| n = 0 |
| refs = [] |
| |
| for ref in all_refs: |
| refs.append(ref) |
| |
| n += 1 |
| if n >= chunk_size: |
| yield refs |
| n = 0 |
| refs = [] |
| |
| yield refs |
| |
| def check_undefined_symbols(self, max_workers=None, chunk_size=50, |
| found=None, dry_run=None): |
| """Seach ABI for sysfs symbols missing documentation""" |
| |
| self.abi.parse_abi() |
| |
| if self.abi.debug & AbiDebug.GRAPH: |
| self.print_graph() |
| |
| all_refs = [] |
| for ref in self._ref_interactor(self.root): |
| all_refs.append(ref["__name"]) |
| |
| if dry_run: |
| print("Would check", file=sys.stderr) |
| for ref in all_refs: |
| print(", ".join(ref)) |
| |
| return |
| |
| print("Starting to search symbols (it may take several minutes):", |
| file=sys.stderr) |
| start = datetime.now() |
| old_elapsed = None |
| |
| # Python doesn't support multithreading due to limitations on its |
| # global lock (GIL). While Python 3.13 finally made GIL optional, |
| # there are still issues related to it. Also, we want to have |
| # backward compatibility with older versions of Python. |
| # |
| # So, use instead multiprocess. However, Python is very slow passing |
| # data from/to multiple processes. Also, it may consume lots of memory |
| # if the data to be shared is not small. So, we need to group workload |
| # in chunks that are big enough to generate performance gains while |
| # not being so big that would cause out-of-memory. |
| |
| num_refs = len(all_refs) |
| print(f"Number of references to parse: {num_refs}", file=sys.stderr) |
| |
| if not max_workers: |
| max_workers = os.cpu_count() |
| elif max_workers > os.cpu_count(): |
| max_workers = os.cpu_count() |
| |
| max_workers = max(max_workers, 1) |
| |
| max_chunk_size = int((num_refs + max_workers - 1) / max_workers) |
| chunk_size = min(chunk_size, max_chunk_size) |
| chunk_size = max(1, chunk_size) |
| |
| if max_workers > 1: |
| executor = futures.ProcessPoolExecutor |
| |
| # Place references in a random order. This may help improving |
| # performance, by mixing complex/simple expressions when creating |
| # chunks |
| shuffle(all_refs) |
| else: |
| # Python has a high overhead with processes. When there's just |
| # one worker, it is faster to not create a new process. |
| # Yet, User still deserves to have a progress print. So, use |
| # python's "thread", which is actually a single process, using |
| # an internal schedule to switch between tasks. No performance |
| # gains for non-IO tasks, but still it can be quickly interrupted |
| # from time to time to display progress. |
| executor = futures.ThreadPoolExecutor |
| |
| not_found = [] |
| f_list = [] |
| with executor(max_workers=max_workers) as exe: |
| for refs in self.get_fileref(all_refs, chunk_size): |
| if refs: |
| try: |
| f_list.append(exe.submit(self.check_file, refs, found)) |
| |
| except KeyboardInterrupt: |
| return |
| |
| total = len(f_list) |
| |
| if not total: |
| if self.abi.re_string: |
| print(f"No ABI symbol matches {self.abi.search_string}") |
| else: |
| self.abi.log.warning("No ABI symbols found") |
| return |
| |
| print(f"{len(f_list):6d} jobs queued on {max_workers} workers", |
| file=sys.stderr) |
| |
| while f_list: |
| try: |
| t = futures.wait(f_list, timeout=1, |
| return_when=futures.FIRST_COMPLETED) |
| |
| done = t[0] |
| |
| for fut in done: |
| res_list = fut.result() |
| |
| for res in res_list: |
| if not res["found"]: |
| not_found.append(res["fname"]) |
| if res["msg"]: |
| print(res["msg"]) |
| |
| f_list.remove(fut) |
| except KeyboardInterrupt: |
| return |
| |
| except RuntimeError as e: |
| self.abi.log.warning(f"Future: {e}") |
| break |
| |
| if sys.stderr.isatty(): |
| elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0] |
| if len(f_list) < total: |
| elapsed += f" ({total - len(f_list)}/{total} jobs completed). " |
| if elapsed != old_elapsed: |
| print(elapsed + "\r", end="", flush=True, |
| file=sys.stderr) |
| old_elapsed = elapsed |
| |
| elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0] |
| print(elapsed, file=sys.stderr) |
| |
| for f in sorted(not_found): |
| print(f"{f} not found.") |
