| # resulttool - report test results in JUnit XML format |
| # |
| # Copyright (c) 2024, Siemens AG. |
| # |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| |
| import os |
| import xml.etree.ElementTree as ET |
| import resulttool.resultutils as resultutils |
| |
| |
| DEFAULT_JUNIT_FILE = "junit.xml" |
| |
| |
| class PtestSummary: |
| """Collected infromation of one ptest suite |
| |
| Collect the information of many ptests of a ptest suite such as ptestresults.APtest.test_foo1, |
| ptestresults.APtest.test_foo2, ... as one testcase APtest. This can be merged into information |
| from ptestresult.sections. |
| """ |
| def __init__(self): |
| self.tests = 0 |
| self.ERROR = [] |
| self.FAILED = [] |
| self.SKIPPED = [] |
| |
| def add_status(self, ptest_testcase, status): |
| self.tests += 1 |
| if status == "FAILED": |
| self.FAILED.append(ptest_testcase) |
| elif status == "ERROR": |
| self.ERROR.append(ptest_testcase) |
| elif status == "SKIPPED": |
| self.SKIPPED.append(ptest_testcase) |
| |
| @property |
| def status(self): |
| """Normalize the status of many ptests to one status of the ptest suite""" |
| if len(self.ERROR) > 0: |
| return "ERROR" |
| if len(self.FAILED) > 0: |
| return "FAILED" |
| if len(self.SKIPPED) == self.tests: |
| return "SKIPPED" |
| return "SUCCESS" |
| |
| @property |
| def log_summary(self): |
| """Return a summary of the ptest suite""" |
| summary_parts = [] |
| |
| if self.ERROR: |
| summary_parts.append("ERROR:" + os.linesep) |
| summary_parts.append(os.linesep.join(["- " + s for s in self.ERROR]) + os.linesep) |
| |
| if self.FAILED: |
| summary_parts.append("FAILED:" + os.linesep) |
| summary_parts.append(os.linesep.join(["- " + s for s in self.FAILED]) + os.linesep) |
| |
| if self.SKIPPED: |
| summary_parts.append("SKIPPED:" + os.linesep) |
| summary_parts.append(os.linesep.join(["- " + s for s in self.SKIPPED]) + os.linesep) |
| |
| return "".join(summary_parts) if summary_parts else "No failures or errors" |
| |
| |
| def create_testcase(testsuite, testcase_dict, status, status_message, status_text=None, system_out=None): |
| """Create a junit testcase node""" |
| testcase_node = ET.SubElement(testsuite, "testcase", testcase_dict) |
| |
| print("%s -> %s status: %s" % (testcase_dict["classname"], testcase_dict["name"], status)) |
| |
| se = None |
| if status == "SKIPPED": |
| se = ET.SubElement(testcase_node, "skipped", message = status_message.replace('\n', ' ') if status_message else None) |
| elif status == "FAILED": |
| se = ET.SubElement(testcase_node, "failure") |
| se = ET.SubElement(testcase_node, "system-out").text = (status_message or "") + os.linesep + (system_out or "") |
| elif status == "ERROR": |
| se = ET.SubElement(testcase_node, "error") |
| se = ET.SubElement(testcase_node, "system-out").text = (status_message or "") + os.linesep + (system_out or "") |
| |
| def junit_tree(testresults, test_log_dir=None): |
| """Create a JUnit XML tree from testresults |
| |
| Generates a tuple of the XML tree and a dictionary of ptest log files. |
| The dictionary contains the path where the log file is located as key and the log content as value. |
| The log file path is test_log_dir/ptest_name.log. |
| """ |
| test_logfiles = {} |
| testsuites_node = ET.Element("testsuites") |
| |
| image_errors = image_failures = image_skipped = image_tests = image_total_time = 0 |
| ptest_errors = ptest_failures = ptest_skipped = ptest_tests = ptest_total_time = 0 |
| total_errors = total_failures = total_skipped = total_tests = total_time = 0 |
| |
| for _, run_name, _, results in resultutils.test_run_results(testresults): |
| test_run_testsuite = ET.SubElement(testsuites_node, "testsuite", name=run_name) |
| |
| # Handle all image tests but skip all ptests related sections |
| imagetest_testsuite = ET.SubElement(test_run_testsuite, "testsuite", name="Image Tests") |
| |
| ptest_summarys = {} |
| |
| for result_id, result in results.items(): |
| if result_id.startswith("ptestresult.sections") or result_id.startswith("ptestresult.rawlogs"): |
| continue |
| |
| if result_id.startswith("ptestresult."): |
| ptest_name = result_id.split(".", 3)[1] |
| test_case = result_id.split(".", 3)[2] |
| if ptest_name not in ptest_summarys: |
| ptest_summarys[ptest_name] = PtestSummary() |
| ptest_summarys[ptest_name].add_status(test_case, result["status"]) |
| else: |
| image_total_time += int(result["duration"]) |
| image_tests += 1 |
| status = result["status"] |
| if status == "FAILED": |
| image_failures += 1 |
| elif status == "ERROR": |
| image_errors += 1 |
| elif status == "SKIPPED": |
| image_skipped += 1 |
| |
| testcase_dict = { |
| "name": result_id, |
| "classname": "testimage", |
| "time": str(result["duration"]), |
| } |
| create_testcase( |
| imagetest_testsuite, |
| testcase_dict, |
| status, |
| result.get("log", None)) |
| |
| imagetest_testsuite.set("errors", str(image_errors)) |
| imagetest_testsuite.set("failures", str(image_failures)) |
| imagetest_testsuite.set("skipped", str(image_skipped)) |
| imagetest_testsuite.set("tests", str(image_tests)) |
| imagetest_testsuite.set("time", str(image_total_time)) |
| |
| # Handle all ptest related sections |
| if "ptestresult.sections" in results: |
| ptest_testsuite = ET.SubElement(test_run_testsuite, "testsuite", name="Package Tests") |
| |
| for ptest_name, result in results["ptestresult.sections"].items(): |
| testcase_dict = { |
| "name": ptest_name, |
| "classname": "ptest", |
| "time": str(result["duration"]), |
| } |
| |
| exitcode = result.get("exitcode") |
| log = result.get("log") |
| system_out = None |
| if log: |
| if test_log_dir is not None: |
| test_log_file = os.path.join(test_log_dir, ptest_name + ".log") |
| system_out = f"[[ATTACHMENT|{test_log_file}]]" |
| test_logfiles[test_log_file] = log |
| else: |
| system_out = log |
| |
| # Determine status and log summary |
| if ptest_name in ptest_summarys: |
| status = ptest_summarys[ptest_name].status |
| log_summary = ptest_summarys[ptest_name].log_summary |
| else: |
| # When there is no detailed result for the ptest, we assume it was skipped or errored |
| status = "SKIPPED" if exitcode in (None, "0") else "ERROR" |
| print("Warning: ptest %s has no detailed results, marking as %s" % (ptest_name, status)) |
| log_summary = log if log else "No log available." |
| |
| create_testcase(ptest_testsuite, |
| testcase_dict, |
| status, |
| log_summary, |
| system_out=system_out) |
| |
| ptest_total_time += int(result["duration"]) |
| ptest_tests += 1 |
| if status == "FAILED": |
| ptest_failures += 1 |
| elif status == "ERROR": |
| ptest_errors += 1 |
| elif status == "SKIPPED": |
| ptest_skipped += 1 |
| |
| ptest_testsuite.set("errors", str(ptest_errors)) |
| ptest_testsuite.set("failures", str(ptest_failures)) |
| ptest_testsuite.set("skipped", str(ptest_skipped)) |
| ptest_testsuite.set("tests", str(ptest_tests)) |
| ptest_testsuite.set("time", str(ptest_total_time)) |
| |
| total_errors += image_errors + ptest_errors |
| total_failures += image_failures + ptest_failures |
| total_skipped += image_skipped + ptest_skipped |
| total_tests += image_tests + ptest_tests |
| total_time += image_total_time + ptest_total_time |
| |
| testsuites_node.set("errors", str(total_errors)) |
| testsuites_node.set("failures", str(total_failures)) |
| testsuites_node.set("skipped", str(total_skipped)) |
| testsuites_node.set("tests", str(total_tests)) |
| testsuites_node.set("time", str(total_time)) |
| |
| ptest_success = ptest_tests - ptest_errors - ptest_failures - ptest_skipped |
| image_success = image_tests - image_errors - image_failures - image_skipped |
| total_success = total_tests - total_errors - total_failures - total_skipped |
| |
| print("ptest -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % |
| (ptest_tests, ptest_success, ptest_errors, ptest_failures, ptest_skipped, ptest_total_time)) |
| |
| print("testimage -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % |
| (image_tests, image_success, image_errors, image_failures, image_skipped, image_total_time)) |
| |
| print("total -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % |
| (total_tests, total_success, total_errors, total_failures, total_skipped, total_time)) |
| tree = ET.ElementTree(testsuites_node) |
| return tree, test_logfiles |
| |
| |
| def junit(args, logger): |
| if args.junit_xml_path is not None: |
| junit_xml_path = args.junit_xml_path |
| else: |
| junit_xml_path = os.path.join(os.path.dirname(args.json_file), DEFAULT_JUNIT_FILE) |
| logger.debug("Generating JUnit XML report from %s" % args.json_file) |
| testresults = resultutils.load_resultsdata(args.json_file, configmap=resultutils.store_map) |
| |
| # dump ptest logs to a file in a subdirectory where the junit.xml is located |
| test_log_dir = None |
| if args.attach_log_files: |
| test_log_dir = "test-logs" |
| junit_dir_abs = os.path.dirname(os.path.abspath(junit_xml_path)) |
| test_log_dir_abs = os.path.join(junit_dir_abs, test_log_dir) |
| if not os.path.exists(test_log_dir_abs): |
| os.makedirs(test_log_dir_abs) |
| |
| tree, test_logfiles = junit_tree(testresults, test_log_dir) |
| |
| for test_logfile, log in test_logfiles.items(): |
| with open(os.path.join(junit_dir_abs, test_logfile), "w") as f: |
| f.write(log) |
| |
| tree.write(junit_xml_path, encoding="UTF-8", xml_declaration=True) |
| logger.info("Saved JUnit XML report as %s" % junit_xml_path) |
| |
| |
| def register_commands(subparsers): |
| """Register subcommands from this plugin""" |
| parser_build = subparsers.add_parser("junit", |
| help="create test report in JUnit XML format", |
| description="generate unit test report in JUnit XML format based on the latest test results in the testresults.json.", |
| group="analysis" |
| ) |
| parser_build.set_defaults(func=junit) |
| |
| # If BUILDDIR is set, use the default path for the testresults.json and junit.xml |
| # Otherwise require the user to provide the path to the testresults.json |
| help_json_file = "json file should point to the testresults.json" |
| help_junit_xml_path = "junit xml path allows setting the path of the generated test report." |
| try: |
| builddir = os.environ["BUILDDIR"] |
| except KeyError: |
| builddir = None |
| if builddir: |
| log_path = os.path.join(builddir, "tmp", "log", "oeqa") |
| parser_build.add_argument("json_file", nargs="?", |
| default=os.path.join(log_path, "testresults.json"), |
| help=help_json_file + " (default: %(default)s)") |
| parser_build.add_argument("-j", "--junit_xml_path", |
| default=os.path.join(log_path, DEFAULT_JUNIT_FILE), |
| help=help_junit_xml_path + " (default: %(default)s)") |
| else: |
| parser_build.add_argument("json_file", help=help_json_file) |
| parser_build.add_argument("-j", "--junit_xml_path", nargs="?", |
| help=help_junit_xml_path + " (default: junit.xml in the same folder as the testresults.json)") |
| |
| parser_build.add_argument("-a", "--attach-log-files", action="store_true", default=False, |
| help="Write the log files to subfolder in the same folder as the junit.xml and reference them in the junit.xml") |
