| # resulttool - report test results in JUnit XML format | |
| # | |
| # Copyright (c) 2024, Siemens AG. | |
| # | |
| # SPDX-License-Identifier: GPL-2.0-only | |
| # | |
| import os | |
| import xml.etree.ElementTree as ET | |
| import resulttool.resultutils as resultutils | |
| DEFAULT_JUNIT_FILE = "junit.xml" | |
| class PtestSummary: | |
| """Collected infromation of one ptest suite | |
| Collect the information of many ptests of a ptest suite such as ptestresults.APtest.test_foo1, | |
| ptestresults.APtest.test_foo2, ... as one testcase APtest. This can be merged into information | |
| from ptestresult.sections. | |
| """ | |
| def __init__(self): | |
| self.tests = 0 | |
| self.ERROR = [] | |
| self.FAILED = [] | |
| self.SKIPPED = [] | |
| def add_status(self, ptest_testcase, status): | |
| self.tests += 1 | |
| if status == "FAILED": | |
| self.FAILED.append(ptest_testcase) | |
| elif status == "ERROR": | |
| self.ERROR.append(ptest_testcase) | |
| elif status == "SKIPPED": | |
| self.SKIPPED.append(ptest_testcase) | |
| @property | |
| def status(self): | |
| """Normalize the status of many ptests to one status of the ptest suite""" | |
| if len(self.ERROR) > 0: | |
| return "ERROR" | |
| if len(self.FAILED) > 0: | |
| return "FAILED" | |
| if len(self.SKIPPED) == self.tests: | |
| return "SKIPPED" | |
| return "SUCCESS" | |
| @property | |
| def log_summary(self): | |
| """Return a summary of the ptest suite""" | |
| summary_parts = [] | |
| if self.ERROR: | |
| summary_parts.append("ERROR:" + os.linesep) | |
| summary_parts.append(os.linesep.join(["- " + s for s in self.ERROR]) + os.linesep) | |
| if self.FAILED: | |
| summary_parts.append("FAILED:" + os.linesep) | |
| summary_parts.append(os.linesep.join(["- " + s for s in self.FAILED]) + os.linesep) | |
| if self.SKIPPED: | |
| summary_parts.append("SKIPPED:" + os.linesep) | |
| summary_parts.append(os.linesep.join(["- " + s for s in self.SKIPPED]) + os.linesep) | |
| return "".join(summary_parts) if summary_parts else "No failures or errors" | |
| def create_testcase(testsuite, testcase_dict, status, status_message, status_text=None, system_out=None): | |
| """Create a junit testcase node""" | |
| testcase_node = ET.SubElement(testsuite, "testcase", testcase_dict) | |
| print("%s -> %s status: %s" % (testcase_dict["classname"], testcase_dict["name"], status)) | |
| se = None | |
| if status == "SKIPPED": | |
| se = ET.SubElement(testcase_node, "skipped", message = status_message.replace('\n', ' ') if status_message else None) | |
| elif status == "FAILED": | |
| se = ET.SubElement(testcase_node, "failure") | |
| se = ET.SubElement(testcase_node, "system-out").text = (status_message or "") + os.linesep + (system_out or "") | |
| elif status == "ERROR": | |
| se = ET.SubElement(testcase_node, "error") | |
| se = ET.SubElement(testcase_node, "system-out").text = (status_message or "") + os.linesep + (system_out or "") | |
| def junit_tree(testresults, test_log_dir=None): | |
| """Create a JUnit XML tree from testresults | |
| Generates a tuple of the XML tree and a dictionary of ptest log files. | |
| The dictionary contains the path where the log file is located as key and the log content as value. | |
| The log file path is test_log_dir/ptest_name.log. | |
| """ | |
| test_logfiles = {} | |
| testsuites_node = ET.Element("testsuites") | |
| image_errors = image_failures = image_skipped = image_tests = image_total_time = 0 | |
| ptest_errors = ptest_failures = ptest_skipped = ptest_tests = ptest_total_time = 0 | |
| total_errors = total_failures = total_skipped = total_tests = total_time = 0 | |
| for _, run_name, _, results in resultutils.test_run_results(testresults): | |
| test_run_testsuite = ET.SubElement(testsuites_node, "testsuite", name=run_name) | |
| # Handle all image tests but skip all ptests related sections | |
| imagetest_testsuite = ET.SubElement(test_run_testsuite, "testsuite", name="Image Tests") | |
| ptest_summarys = {} | |
| for result_id, result in results.items(): | |
| if result_id.startswith("ptestresult.sections") or result_id.startswith("ptestresult.rawlogs"): | |
| continue | |
| if result_id.startswith("ptestresult."): | |
| ptest_name = result_id.split(".", 3)[1] | |
| test_case = result_id.split(".", 3)[2] | |
| if ptest_name not in ptest_summarys: | |
| ptest_summarys[ptest_name] = PtestSummary() | |
| ptest_summarys[ptest_name].add_status(test_case, result["status"]) | |
| else: | |
| image_total_time += int(result["duration"]) | |
| image_tests += 1 | |
| status = result["status"] | |
| if status == "FAILED": | |
| image_failures += 1 | |
| elif status == "ERROR": | |
| image_errors += 1 | |
| elif status == "SKIPPED": | |
| image_skipped += 1 | |
| testcase_dict = { | |
| "name": result_id, | |
| "classname": "testimage", | |
| "time": str(result["duration"]), | |
| } | |
| create_testcase( | |
| imagetest_testsuite, | |
| testcase_dict, | |
| status, | |
| result.get("log", None)) | |
| imagetest_testsuite.set("errors", str(image_errors)) | |
| imagetest_testsuite.set("failures", str(image_failures)) | |
| imagetest_testsuite.set("skipped", str(image_skipped)) | |
| imagetest_testsuite.set("tests", str(image_tests)) | |
| imagetest_testsuite.set("time", str(image_total_time)) | |
| # Handle all ptest related sections | |
| if "ptestresult.sections" in results: | |
| ptest_testsuite = ET.SubElement(test_run_testsuite, "testsuite", name="Package Tests") | |
| for ptest_name, result in results["ptestresult.sections"].items(): | |
| testcase_dict = { | |
| "name": ptest_name, | |
| "classname": "ptest", | |
| "time": str(result["duration"]), | |
| } | |
| exitcode = result.get("exitcode") | |
| log = result.get("log") | |
| system_out = None | |
| if log: | |
| if test_log_dir is not None: | |
| test_log_file = os.path.join(test_log_dir, ptest_name + ".log") | |
| system_out = f"[[ATTACHMENT|{test_log_file}]]" | |
| test_logfiles[test_log_file] = log | |
| else: | |
| system_out = log | |
| # Determine status and log summary | |
| if ptest_name in ptest_summarys: | |
| status = ptest_summarys[ptest_name].status | |
| log_summary = ptest_summarys[ptest_name].log_summary | |
| else: | |
| # When there is no detailed result for the ptest, we assume it was skipped or errored | |
| status = "SKIPPED" if exitcode in (None, "0") else "ERROR" | |
| print("Warning: ptest %s has no detailed results, marking as %s" % (ptest_name, status)) | |
| log_summary = log if log else "No log available." | |
| create_testcase(ptest_testsuite, | |
| testcase_dict, | |
| status, | |
| log_summary, | |
| system_out=system_out) | |
| ptest_total_time += int(result["duration"]) | |
| ptest_tests += 1 | |
| if status == "FAILED": | |
| ptest_failures += 1 | |
| elif status == "ERROR": | |
| ptest_errors += 1 | |
| elif status == "SKIPPED": | |
| ptest_skipped += 1 | |
| ptest_testsuite.set("errors", str(ptest_errors)) | |
| ptest_testsuite.set("failures", str(ptest_failures)) | |
| ptest_testsuite.set("skipped", str(ptest_skipped)) | |
| ptest_testsuite.set("tests", str(ptest_tests)) | |
| ptest_testsuite.set("time", str(ptest_total_time)) | |
| total_errors += image_errors + ptest_errors | |
| total_failures += image_failures + ptest_failures | |
| total_skipped += image_skipped + ptest_skipped | |
| total_tests += image_tests + ptest_tests | |
| total_time += image_total_time + ptest_total_time | |
| testsuites_node.set("errors", str(total_errors)) | |
| testsuites_node.set("failures", str(total_failures)) | |
| testsuites_node.set("skipped", str(total_skipped)) | |
| testsuites_node.set("tests", str(total_tests)) | |
| testsuites_node.set("time", str(total_time)) | |
| ptest_success = ptest_tests - ptest_errors - ptest_failures - ptest_skipped | |
| image_success = image_tests - image_errors - image_failures - image_skipped | |
| total_success = total_tests - total_errors - total_failures - total_skipped | |
| print("ptest -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % | |
| (ptest_tests, ptest_success, ptest_errors, ptest_failures, ptest_skipped, ptest_total_time)) | |
| print("testimage -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % | |
| (image_tests, image_success, image_errors, image_failures, image_skipped, image_total_time)) | |
| print("total -> tests: %d, success: %d, error: %d, failures: %d, skipped: %d, time: %d" % | |
| (total_tests, total_success, total_errors, total_failures, total_skipped, total_time)) | |
| tree = ET.ElementTree(testsuites_node) | |
| return tree, test_logfiles | |
| def junit(args, logger): | |
| if args.junit_xml_path is not None: | |
| junit_xml_path = args.junit_xml_path | |
| else: | |
| junit_xml_path = os.path.join(os.path.dirname(args.json_file), DEFAULT_JUNIT_FILE) | |
| logger.debug("Generating JUnit XML report from %s" % args.json_file) | |
| testresults = resultutils.load_resultsdata(args.json_file, configmap=resultutils.store_map) | |
| # dump ptest logs to a file in a subdirectory where the junit.xml is located | |
| test_log_dir = None | |
| if args.attach_log_files: | |
| test_log_dir = "test-logs" | |
| junit_dir_abs = os.path.dirname(os.path.abspath(junit_xml_path)) | |
| test_log_dir_abs = os.path.join(junit_dir_abs, test_log_dir) | |
| if not os.path.exists(test_log_dir_abs): | |
| os.makedirs(test_log_dir_abs) | |
| tree, test_logfiles = junit_tree(testresults, test_log_dir) | |
| for test_logfile, log in test_logfiles.items(): | |
| with open(os.path.join(junit_dir_abs, test_logfile), "w") as f: | |
| f.write(log) | |
| tree.write(junit_xml_path, encoding="UTF-8", xml_declaration=True) | |
| logger.info("Saved JUnit XML report as %s" % junit_xml_path) | |
| def register_commands(subparsers): | |
| """Register subcommands from this plugin""" | |
| parser_build = subparsers.add_parser("junit", | |
| help="create test report in JUnit XML format", | |
| description="generate unit test report in JUnit XML format based on the latest test results in the testresults.json.", | |
| group="analysis" | |
| ) | |
| parser_build.set_defaults(func=junit) | |
| # If BUILDDIR is set, use the default path for the testresults.json and junit.xml | |
| # Otherwise require the user to provide the path to the testresults.json | |
| help_json_file = "json file should point to the testresults.json" | |
| help_junit_xml_path = "junit xml path allows setting the path of the generated test report." | |
| try: | |
| builddir = os.environ["BUILDDIR"] | |
| except KeyError: | |
| builddir = None | |
| if builddir: | |
| log_path = os.path.join(builddir, "tmp", "log", "oeqa") | |
| parser_build.add_argument("json_file", nargs="?", | |
| default=os.path.join(log_path, "testresults.json"), | |
| help=help_json_file + " (default: %(default)s)") | |
| parser_build.add_argument("-j", "--junit_xml_path", | |
| default=os.path.join(log_path, DEFAULT_JUNIT_FILE), | |
| help=help_junit_xml_path + " (default: %(default)s)") | |
| else: | |
| parser_build.add_argument("json_file", help=help_json_file) | |
| parser_build.add_argument("-j", "--junit_xml_path", nargs="?", | |
| help=help_junit_xml_path + " (default: junit.xml in the same folder as the testresults.json)") | |
| parser_build.add_argument("-a", "--attach-log-files", action="store_true", default=False, | |
| help="Write the log files to subfolder in the same folder as the junit.xml and reference them in the junit.xml") |