| #!/usr/bin/env python3 |
| # pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302 |
| # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. |
| # SPDX-License-Identifier: GPL-2.0 |
| |
| """ |
| Parse ABI documentation and produce results from it. |
| """ |
| |
| from argparse import Namespace |
| import logging |
| import os |
| import re |
| |
| from pprint import pformat |
| from random import randrange, seed |
| |
| # Import Python modules |
| |
| from helpers import AbiDebug, ABI_DIR |
| |
| |
| class AbiParser: |
| """Main class to parse ABI files""" |
| |
| TAGS = r"(what|where|date|kernelversion|contact|description|users)" |
| XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)" |
| |
| def __init__(self, directory, logger=None, |
| enable_lineno=False, show_warnings=True, debug=0): |
| """Stores arguments for the class and initialize class vars""" |
| |
| self.directory = directory |
| self.enable_lineno = enable_lineno |
| self.show_warnings = show_warnings |
| self.debug = debug |
| |
| if not logger: |
| self.log = logging.getLogger("get_abi") |
| else: |
| self.log = logger |
| |
| self.data = {} |
| self.what_symbols = {} |
| self.file_refs = {} |
| self.what_refs = {} |
| |
| # Ignore files that contain such suffixes |
| self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~") |
| |
| # Regular expressions used on parser |
| self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR) |
| self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I) |
| self.re_valid = re.compile(self.TAGS) |
| self.re_start_spc = re.compile(r"(\s*)(\S.*)") |
| self.re_whitespace = re.compile(r"^\s+") |
| |
| # Regular used on print |
| self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})") |
| self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])") |
| self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)") |
| self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n") |
| self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst") |
| self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)") |
| self.re_xref_node = re.compile(self.XREF) |
| |
| def warn(self, fdata, msg, extra=None): |
| """Displays a parse error if warning is enabled""" |
| |
| if not self.show_warnings: |
| return |
| |
| msg = f"{fdata.fname}:{fdata.ln}: {msg}" |
| if extra: |
| msg += "\n\t\t" + extra |
| |
| self.log.warning(msg) |
| |
| def add_symbol(self, what, fname, ln=None, xref=None): |
| """Create a reference table describing where each 'what' is located""" |
| |
| if what not in self.what_symbols: |
| self.what_symbols[what] = {"file": {}} |
| |
| if fname not in self.what_symbols[what]["file"]: |
| self.what_symbols[what]["file"][fname] = [] |
| |
| if ln and ln not in self.what_symbols[what]["file"][fname]: |
| self.what_symbols[what]["file"][fname].append(ln) |
| |
| if xref: |
| self.what_symbols[what]["xref"] = xref |
| |
| def _parse_line(self, fdata, line): |
| """Parse a single line of an ABI file""" |
| |
| new_what = False |
| new_tag = False |
| content = None |
| |
| match = self.re_tag.match(line) |
| if match: |
| new = match.group(1).lower() |
| sep = match.group(2) |
| content = match.group(3) |
| |
| match = self.re_valid.search(new) |
| if match: |
| new_tag = match.group(1) |
| else: |
| if fdata.tag == "description": |
| # New "tag" is actually part of description. |
| # Don't consider it a tag |
| new_tag = False |
| elif fdata.tag != "": |
| self.warn(fdata, f"tag '{fdata.tag}' is invalid", line) |
| |
| if new_tag: |
| # "where" is Invalid, but was a common mistake. Warn if found |
| if new_tag == "where": |
| self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead") |
| new_tag = "what" |
| |
| if new_tag == "what": |
| fdata.space = None |
| |
| if content not in self.what_symbols: |
| self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln) |
| |
| if fdata.tag == "what": |
| fdata.what.append(content.strip("\n")) |
| else: |
| if fdata.key: |
| if "description" not in self.data.get(fdata.key, {}): |
| self.warn(fdata, f"{fdata.key} doesn't have a description") |
| |
| for w in fdata.what: |
| self.add_symbol(what=w, fname=fdata.fname, |
| ln=fdata.what_ln, xref=fdata.key) |
| |
| fdata.label = content |
| new_what = True |
| |
| key = "abi_" + content.lower() |
| fdata.key = self.re_unprintable.sub("_", key).strip("_") |
| |
| # Avoid duplicated keys but using a defined seed, to make |
| # the namespace identical if there aren't changes at the |
| # ABI symbols |
| seed(42) |
| |
| while fdata.key in self.data: |
| char = randrange(0, 51) + ord("A") |
| if char > ord("Z"): |
| char += ord("a") - ord("Z") - 1 |
| |
| fdata.key += chr(char) |
| |
| if fdata.key and fdata.key not in self.data: |
| self.data[fdata.key] = { |
| "what": [content], |
| "file": [fdata.file_ref], |
| "path": fdata.ftype, |
| "line_no": fdata.ln, |
| } |
| |
| fdata.what = self.data[fdata.key]["what"] |
| |
| self.what_refs[content] = fdata.key |
| fdata.tag = new_tag |
| fdata.what_ln = fdata.ln |
| |
| if fdata.nametag["what"]: |
| t = (content, fdata.key) |
| if t not in fdata.nametag["symbols"]: |
| fdata.nametag["symbols"].append(t) |
| |
| return |
| |
| if fdata.tag and new_tag: |
| fdata.tag = new_tag |
| |
| if new_what: |
| fdata.label = "" |
| |
| if "description" in self.data[fdata.key]: |
| self.data[fdata.key]["description"] += "\n\n" |
| |
| if fdata.file_ref not in self.data[fdata.key]["file"]: |
| self.data[fdata.key]["file"].append(fdata.file_ref) |
| |
| if self.debug == AbiDebug.WHAT_PARSING: |
| self.log.debug("what: %s", fdata.what) |
| |
| if not fdata.what: |
| self.warn(fdata, "'What:' should come first:", line) |
| return |
| |
| if new_tag == "description": |
| fdata.space = None |
| |
| if content: |
| sep = sep.replace(":", " ") |
| |
| c = " " * len(new_tag) + sep + content |
| c = c.expandtabs() |
| |
| match = self.re_start_spc.match(c) |
| if match: |
| # Preserve initial spaces for the first line |
| fdata.space = match.group(1) |
| content = match.group(2) + "\n" |
| |
| self.data[fdata.key][fdata.tag] = content |
| |
| return |
| |
| # Store any contents before tags at the database |
| if not fdata.tag and "what" in fdata.nametag: |
| fdata.nametag["description"] += line |
| return |
| |
| if fdata.tag == "description": |
| content = line.expandtabs() |
| |
| if self.re_whitespace.sub("", content) == "": |
| self.data[fdata.key][fdata.tag] += "\n" |
| return |
| |
| if fdata.space is None: |
| match = self.re_start_spc.match(content) |
| if match: |
| # Preserve initial spaces for the first line |
| fdata.space = match.group(1) |
| |
| content = match.group(2) + "\n" |
| else: |
| if content.startswith(fdata.space): |
| content = content[len(fdata.space):] |
| |
| else: |
| fdata.space = "" |
| |
| if fdata.tag == "what": |
| w = content.strip("\n") |
| if w: |
| self.data[fdata.key][fdata.tag].append(w) |
| else: |
| self.data[fdata.key][fdata.tag] += content |
| return |
| |
| content = line.strip() |
| if fdata.tag: |
| if fdata.tag == "what": |
| w = content.strip("\n") |
| if w: |
| self.data[fdata.key][fdata.tag].append(w) |
| else: |
| self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n") |
| return |
| |
| # Everything else is error |
| if content: |
| self.warn(fdata, "Unexpected content", line) |
| |
| def parse_readme(self, nametag, fname): |
| """Parse ABI README file""" |
| |
| nametag["what"] = ["Introduction"] |
| nametag["path"] = "README" |
| with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: |
| for line in fp: |
| match = self.re_tag.match(line) |
| if match: |
| new = match.group(1).lower() |
| |
| match = self.re_valid.search(new) |
| if match: |
| nametag["description"] += "\n:" + line |
| continue |
| |
| nametag["description"] += line |
| |
| def parse_file(self, fname, path, basename): |
| """Parse a single file""" |
| |
| ref = f"abi_file_{path}_{basename}" |
| ref = self.re_unprintable.sub("_", ref).strip("_") |
| |
| # Store per-file state into a namespace variable. This will be used |
| # by the per-line parser state machine and by the warning function. |
| fdata = Namespace |
| |
| fdata.fname = fname |
| fdata.name = basename |
| |
| pos = fname.find(ABI_DIR) |
| if pos > 0: |
| f = fname[pos:] |
| else: |
| f = fname |
| |
| fdata.file_ref = (f, ref) |
| self.file_refs[f] = ref |
| |
| fdata.ln = 0 |
| fdata.what_ln = 0 |
| fdata.tag = "" |
| fdata.label = "" |
| fdata.what = [] |
| fdata.key = None |
| fdata.xrefs = None |
| fdata.space = None |
| fdata.ftype = path.split("/")[0] |
| |
| fdata.nametag = {} |
| fdata.nametag["what"] = [f"ABI file {path}/{basename}"] |
| fdata.nametag["type"] = "File" |
| fdata.nametag["path"] = fdata.ftype |
| fdata.nametag["file"] = [fdata.file_ref] |
| fdata.nametag["line_no"] = 1 |
| fdata.nametag["description"] = "" |
| fdata.nametag["symbols"] = [] |
| |
| self.data[ref] = fdata.nametag |
| |
| if self.debug & AbiDebug.WHAT_OPEN: |
| self.log.debug("Opening file %s", fname) |
| |
| if basename == "README": |
| self.parse_readme(fdata.nametag, fname) |
| return |
| |
| with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: |
| for line in fp: |
| fdata.ln += 1 |
| |
| self._parse_line(fdata, line) |
| |
| if "description" in fdata.nametag: |
| fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n") |
| |
| if fdata.key: |
| if "description" not in self.data.get(fdata.key, {}): |
| self.warn(fdata, f"{fdata.key} doesn't have a description") |
| |
| for w in fdata.what: |
| self.add_symbol(what=w, fname=fname, xref=fdata.key) |
| |
| def _parse_abi(self, root=None): |
| """Internal function to parse documentation ABI recursively""" |
| |
| if not root: |
| root = self.directory |
| |
| with os.scandir(root) as obj: |
| for entry in obj: |
| name = os.path.join(root, entry.name) |
| |
| if entry.is_dir(): |
| self._parse_abi(name) |
| continue |
| |
| if not entry.is_file(): |
| continue |
| |
| basename = os.path.basename(name) |
| |
| if basename.startswith("."): |
| continue |
| |
| if basename.endswith(self.ignore_suffixes): |
| continue |
| |
| path = self.re_abi_dir.sub("", os.path.dirname(name)) |
| |
| self.parse_file(name, path, basename) |
| |
| def parse_abi(self, root=None): |
| """Parse documentation ABI""" |
| |
| self._parse_abi(root) |
| |
| if self.debug & AbiDebug.DUMP_ABI_STRUCTS: |
| self.log.debug(pformat(self.data)) |
| |
| def desc_txt(self, desc): |
| """Print description as found inside ABI files""" |
| |
| desc = desc.strip(" \t\n") |
| |
| return desc + "\n\n" |
| |
| def xref(self, fname): |
| """ |
| Converts a Documentation/ABI + basename into a ReST cross-reference |
| """ |
| |
| xref = self.file_refs.get(fname) |
| if not xref: |
| return None |
| else: |
| return xref |
| |
| def desc_rst(self, desc): |
| """Enrich ReST output by creating cross-references""" |
| |
| # Remove title markups from the description |
| # Having titles inside ABI files will only work if extra |
| # care would be taken in order to strictly follow the same |
| # level order for each markup. |
| desc = self.re_title_mark.sub("\n\n", "\n" + desc) |
| desc = desc.rstrip(" \t\n").lstrip("\n") |
| |
| # Python's regex performance for non-compiled expressions is a lot |
| # than Perl, as Perl automatically caches them at their |
| # first usage. Here, we'll need to do the same, as otherwise the |
| # performance penalty is be high |
| |
| new_desc = "" |
| for d in desc.split("\n"): |
| if d == "": |
| new_desc += "\n" |
| continue |
| |
| # Use cross-references for doc files where needed |
| d = self.re_doc.sub(r":doc:`/\1`", d) |
| |
| # Use cross-references for ABI generated docs where needed |
| matches = self.re_abi.findall(d) |
| for m in matches: |
| abi = m[0] + m[1] |
| |
| xref = self.file_refs.get(abi) |
| if not xref: |
| # This may happen if ABI is on a separate directory, |
| # like parsing ABI testing and symbol is at stable. |
| # The proper solution is to move this part of the code |
| # for it to be inside sphinx/kernel_abi.py |
| self.log.info("Didn't find ABI reference for '%s'", abi) |
| else: |
| new = self.re_escape.sub(r"\\\1", m[1]) |
| d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d) |
| |
| # Seek for cross reference symbols like /sys/... |
| # Need to be careful to avoid doing it on a code block |
| if d[0] not in [" ", "\t"]: |
| matches = self.re_xref_node.findall(d) |
| for m in matches: |
| # Finding ABI here is more complex due to wildcards |
| xref = self.what_refs.get(m) |
| if xref: |
| new = self.re_escape.sub(r"\\\1", m) |
| d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d) |
| |
| new_desc += d + "\n" |
| |
| return new_desc + "\n\n" |
| |
| def doc(self, output_in_txt=False, show_symbols=True, show_file=True, |
| filter_path=None): |
| """Print ABI at stdout""" |
| |
| part = None |
| for key, v in sorted(self.data.items(), |
| key=lambda x: (x[1].get("type", ""), |
| x[1].get("what"))): |
| |
| wtype = v.get("type", "Symbol") |
| file_ref = v.get("file") |
| names = v.get("what", [""]) |
| |
| if wtype == "File": |
| if not show_file: |
| continue |
| else: |
| if not show_symbols: |
| continue |
| |
| if filter_path: |
| if v.get("path") != filter_path: |
| continue |
| |
| msg = "" |
| |
| if wtype != "File": |
| cur_part = names[0] |
| if cur_part.find("/") >= 0: |
| match = self.re_what.match(cur_part) |
| if match: |
| symbol = match.group(1).rstrip("/") |
| cur_part = "Symbols under " + symbol |
| |
| if cur_part and cur_part != part: |
| part = cur_part |
| msg += part + "\n"+ "-" * len(part) +"\n\n" |
| |
| msg += f".. _{key}:\n\n" |
| |
| max_len = 0 |
| for i in range(0, len(names)): # pylint: disable=C0200 |
| names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**" |
| |
| max_len = max(max_len, len(names[i])) |
| |
| msg += "+-" + "-" * max_len + "-+\n" |
| for name in names: |
| msg += f"| {name}" + " " * (max_len - len(name)) + " |\n" |
| msg += "+-" + "-" * max_len + "-+\n" |
| msg += "\n" |
| |
| for ref in file_ref: |
| if wtype == "File": |
| msg += f".. _{ref[1]}:\n\n" |
| else: |
| base = os.path.basename(ref[0]) |
| msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n" |
| |
| if wtype == "File": |
| msg += names[0] +"\n" + "-" * len(names[0]) +"\n\n" |
| |
| desc = v.get("description") |
| if not desc and wtype != "File": |
| msg += f"DESCRIPTION MISSING for {names[0]}\n\n" |
| |
| if desc: |
| if output_in_txt: |
| msg += self.desc_txt(desc) |
| else: |
| msg += self.desc_rst(desc) |
| |
| symbols = v.get("symbols") |
| if symbols: |
| msg += "Has the following ABI:\n\n" |
| |
| for w, label in symbols: |
| # Escape special chars from content |
| content = self.re_escape.sub(r"\\\1", w) |
| |
| msg += f"- :ref:`{content} <{label}>`\n\n" |
| |
| users = v.get("users") |
| if users and users.strip(" \t\n"): |
| users = users.strip("\n").replace('\n', '\n\t') |
| msg += f"Users:\n\t{users}\n\n" |
| |
| ln = v.get("line_no", 1) |
| |
| yield (msg, file_ref[0][0], ln) |
| |
| def check_issues(self): |
| """Warn about duplicated ABI entries""" |
| |
| for what, v in self.what_symbols.items(): |
| files = v.get("file") |
| if not files: |
| # Should never happen if the parser works properly |
| self.log.warning("%s doesn't have a file associated", what) |
| continue |
| |
| if len(files) == 1: |
| continue |
| |
| f = [] |
| for fname, lines in sorted(files.items()): |
| if not lines: |
| f.append(f"{fname}") |
| elif len(lines) == 1: |
| f.append(f"{fname}:{lines[0]}") |
| else: |
| m = fname + "lines " |
| m += ", ".join(str(x) for x in lines) |
| f.append(m) |
| |
| self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f)) |
| |
| def search_symbols(self, expr): |
| """ Searches for ABI symbols """ |
| |
| regex = re.compile(expr, re.I) |
| |
| found_keys = 0 |
| for t in sorted(self.data.items(), key=lambda x: [0]): |
| v = t[1] |
| |
| wtype = v.get("type", "") |
| if wtype == "File": |
| continue |
| |
| for what in v.get("what", [""]): |
| if regex.search(what): |
| found_keys += 1 |
| |
| kernelversion = v.get("kernelversion", "").strip(" \t\n") |
| date = v.get("date", "").strip(" \t\n") |
| contact = v.get("contact", "").strip(" \t\n") |
| users = v.get("users", "").strip(" \t\n") |
| desc = v.get("description", "").strip(" \t\n") |
| |
| files = [] |
| for f in v.get("file", ()): |
| files.append(f[0]) |
| |
| what = str(found_keys) + ". " + what |
| title_tag = "-" * len(what) |
| |
| print(f"\n{what}\n{title_tag}\n") |
| |
| if kernelversion: |
| print(f"Kernel version:\t\t{kernelversion}") |
| |
| if date: |
| print(f"Date:\t\t\t{date}") |
| |
| if contact: |
| print(f"Contact:\t\t{contact}") |
| |
| if users: |
| print(f"Users:\t\t\t{users}") |
| |
| print("Defined on file(s):\t" + ", ".join(files)) |
| |
| if desc: |
| desc = desc.strip("\n") |
| print(f"\n{desc}\n") |
| |
| if not found_keys: |
| print(f"Regular expression /{expr}/ not found.") |
