| QEMU_OPTIONS = "-r ${OLDEST_KERNEL} ${@d.getVar("QEMU_EXTRAOPTIONS:tune-%s" % d.getVar('TUNE_PKGARCH')) or ""}" |
| QEMU_OPTIONS[vardeps] += "QEMU_EXTRAOPTIONS:tune-${TUNE_PKGARCH}" |
| |
| ENABLE_VERSION_MISMATCH_CHECK ?= "${@'1' if bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', True, False, d) else '0'}" |
| DEBUG_VERSION_MISMATCH_CHECK ?= "1" |
| CHECK_VERSION_PV ?= "" |
| |
| DEPENDS:append:class-target = "${@' qemu-native' if bb.utils.to_boolean(d.getVar('ENABLE_VERSION_MISMATCH_CHECK')) else ''}" |
| |
| QEMU_EXEC ?= "${@oe.qemu.qemu_wrapper_cmdline(d, '${STAGING_DIR_HOST}', ['${STAGING_DIR_HOST}${libdir}','${STAGING_DIR_HOST}${base_libdir}', '${PKGD}${libdir}', '${PKGD}${base_libdir}'])}" |
| |
| python do_package_check_version_mismatch() { |
| import re |
| import subprocess |
| import shutil |
| import signal |
| import glob |
| |
| classes_skip = ["nopackage", "image", "native", "cross", "crosssdk", "cross-canadian"] |
| for cs in classes_skip: |
| if bb.data.inherits_class(cs, d): |
| bb.note(f"Skip do_package_check_version_mismatch as {cs} is inherited.") |
| return |
| |
| if not bb.utils.to_boolean(d.getVar('ENABLE_VERSION_MISMATCH_CHECK')): |
| bb.note("Skip do_package_check_version_mismatch as ENABLE_VERSION_MISMATCH_CHECK is disabled.") |
| return |
| |
| __regexp_version_broad_match__ = re.compile(r"(?:\s|^|-|_|/|=| go|\()" + |
| r"(?P<version>v?[0-9][0-9.][0-9+.\-_~\(\)]*?|UNKNOWN)" + |
| r"(?:[+\-]release.*|[+\-]stable.*|)" + |
| r"(?P<extra>[+\-]unknown|[+\-]dirty|[+\-]rc?\d{1,3}|\+cargo-[0-9.]+|" + |
| r"[a-z]|-?[pP][0-9]{1,3}|-?beta[^\s]*|-?alpha[^\s]*|)" + |
| r"(?P<extra2>[+\-]dev|[+\-]devel|)" + |
| r"(?:,|:|\.|\)|-[0-9a-g]{6,42}|)" + |
| r"(?=\s|$)" |
| ) |
| __regexp_exclude_year__ = re.compile(r"^(19|20)[0-9]{2}$") |
| __regexp_single_number_ending_with_dot__ = re.compile(r"^\d\.$") |
| |
| def is_shared_library(filepath): |
| return re.match(r'.*\.so(\.\d+)*$', filepath) is not None |
| |
| def get_possible_versions(output_contents, full_cmd=None, max_lines=None): |
| # |
| # Algorithm: |
| # 1. Check version line by line. |
| # 2. Skip some lines which we know that do not contain version information, e.g., License, Copyright. |
| # 3. Do broad match, finding all possible versions. |
| # 4. If there's a version found by any match, do exclude match (e.g., exclude years) |
| # 5. If there's a valid version, do stripping and converting and then add to possible_versions. |
| # 6. Return possible_versions |
| # |
| possible_versions = [] |
| content_lines = output_contents.split("\n") |
| if max_lines: |
| content_lines = content_lines[0:max_lines] |
| if full_cmd: |
| base_cmd = os.path.basename(full_cmd) |
| __regex_help_format__ = re.compile(r"-[^\s].*") |
| for line in content_lines: |
| line = line.strip() |
| # skip help lines |
| if __regex_help_format__.match(line): |
| continue |
| # avoid command itself affecting output |
| if full_cmd: |
| if line.startswith(base_cmd): |
| line = line[len(base_cmd):] |
| elif line.startswith(full_cmd): |
| line = line[len(full_cmd):] |
| # skip specific lines |
| skip_keywords_start = ["copyright", "license", "compiled", "build", "built"] |
| skip_line = False |
| for sks in skip_keywords_start: |
| if line.lower().startswith(sks): |
| skip_line = True |
| break |
| if skip_line: |
| continue |
| |
| # try broad match |
| for match in __regexp_version_broad_match__.finditer(line): |
| version = match.group("version") |
| #print(f"version = {version}") |
| # do exclude match |
| exclude_match = __regexp_exclude_year__.match(version) |
| if exclude_match: |
| continue |
| exclude_match = __regexp_single_number_ending_with_dot__.match(version) |
| if exclude_match: |
| continue |
| # do some stripping and converting |
| if version.startswith("("): |
| version = version[1:-1] |
| if version.startswith("v"): |
| version = version[1:] |
| if version.endswith(")") and "(" not in version: |
| version = version[:-1] |
| if not version.endswith(")") and "(" in version: |
| version = version.split('(')[0] |
| # handle extra version info |
| version = version + match.group("extra") + match.group("extra2") |
| possible_versions.append(version) |
| return possible_versions |
| |
| def is_version_mismatch(rvs, pv): |
| got_match = False |
| if pv.startswith("git"): |
| return False |
| if "-pre" in pv: |
| pv = pv.split("-pre")[0] |
| if pv.startswith("v"): |
| pv = pv[1:] |
| for rv in rvs: |
| if rv == pv: |
| got_match = True |
| break |
| pv = pv.split("+git")[0] |
| # handle % character in pv which means matching any chars |
| if '%' in pv: |
| escaped_pv = re.escape(pv) |
| regex_pattern = escaped_pv.replace('%', '.*') |
| regex_pattern = f'^{regex_pattern}$' |
| if re.fullmatch(regex_pattern, rv): |
| got_match = True |
| break |
| else: |
| continue |
| # handle cases such as 2.36.0-r0 v.s. 2.36.0 |
| if "-r" in rv: |
| rv = rv.split("-r")[0] |
| chars_to_replace = ["-", "+", "_", "~"] |
| # convert to use "." as the version seperator |
| for cr in chars_to_replace: |
| rv = rv.replace(cr, ".") |
| pv = pv.replace(cr, ".") |
| if rv == pv: |
| got_match = True |
| break |
| # handle case such as 5.2.37(1) v.s. 5.2.37 |
| if "(" in rv: |
| rv = rv.split("(")[0] |
| if rv == pv: |
| got_match = True |
| break |
| # handle case such as 4.4.3p1 |
| if "p" in pv and "p" in rv.lower(): |
| pv = pv.lower().replace(".p", "p") |
| rv = rv.lower().replace(".p", "p") |
| if pv == rv: |
| got_match = True |
| break |
| # handle cases such as 6.00 v.s. 6.0 |
| if rv.startswith(pv): |
| if rv == pv + "0" or rv == pv + ".0": |
| got_match = True |
| break |
| elif pv.startswith(rv): |
| if pv == rv + "0" or pv == rv + ".0": |
| got_match = True |
| break |
| # handle cases such as 21306 v.s. 2.13.6 |
| if "." in pv and not "." in rv: |
| pv_components = pv.split(".") |
| if rv.startswith(pv_components[0]): |
| pv_num = 0 |
| for i in range(0, len(pv_components)): |
| pv_num = pv_num * 100 + int(pv_components[i]) |
| if pv_num == int(rv): |
| got_match = True |
| break |
| if got_match: |
| return False |
| else: |
| return True |
| |
| def is_elf_binary(fexec): |
| fexec_real = os.path.realpath(fexec) |
| elf = oe.qa.ELFFile(fexec_real) |
| try: |
| elf.open() |
| elf.close() |
| return True |
| except: |
| return False |
| |
| def get_shebang(fexec): |
| try: |
| with open(fexec, 'r') as f: |
| first_line = f.readline().strip() |
| if first_line.startswith("#!"): |
| return first_line |
| else: |
| return None |
| except Exception as e: |
| return None |
| |
| def get_interpreter_from_shebang(shebang): |
| if not shebang: |
| return None |
| hosttools_path = d.getVar("TMPDIR") + "/hosttools" |
| if "/sh" in shebang: |
| return hosttools_path + "/sh" |
| elif "/bash" in shebang: |
| return hosttools_path + "/bash" |
| elif "python" in shebang: |
| return hosttools_path + "/python3" |
| elif "perl" in shebang: |
| return hosttools_path + "/perl" |
| else: |
| return None |
| |
| # helper function to get PKGV, useful for recipes such as perf |
| def get_pkgv(pn): |
| pkgdestwork = d.getVar("PKGDESTWORK") |
| recipe_data_fn = pkgdestwork + "/" + pn |
| pn_data = oe.packagedata.read_pkgdatafile(recipe_data_fn) |
| if not "PACKAGES" in pn_data: |
| return d.getVar("PV") |
| packages = pn_data["PACKAGES"].split() |
| for pkg in packages: |
| pkg_fn = pkgdestwork + "/runtime/" + pkg |
| pkg_data = oe.packagedata.read_pkgdatafile(pkg_fn) |
| if "PKGV" in pkg_data: |
| return pkg_data["PKGV"] |
| |
| # |
| # traverse PKGD, find executables and run them to get runtime version information and compare it with recipe version information |
| # |
| enable_debug = bb.utils.to_boolean(d.getVar("DEBUG_VERSION_MISMATCH_CHECK")) |
| pkgd = d.getVar("PKGD") |
| pn = d.getVar("PN") |
| pv = d.getVar("CHECK_VERSION_PV") |
| if not pv: |
| pv = get_pkgv(pn) |
| qemu_exec = d.getVar("QEMU_EXEC").strip() |
| executables = [] |
| possible_versions_all = [] |
| data_lines = [] |
| |
| if enable_debug: |
| debug_directory = d.getVar("TMPDIR") + "/check-version-mismatch" |
| debug_data_file = debug_directory + "/" + pn |
| os.makedirs(debug_directory, exist_ok=True) |
| data_lines.append("pv: %s\n" % pv) |
| |
| # handle a special case: a pure % means matching all, no point in further checking |
| if pv == "%": |
| if enable_debug: |
| data_lines.append("FINAL RESULT: MATCH (%s matches all, skipped)\n\n" % pv) |
| with open(debug_data_file, "w") as f: |
| f.writelines(data_lines) |
| return |
| |
| got_quick_match_result = False |
| # handle python3-xxx recipes quickly |
| __regex_python_module_version__ = re.compile(r"(?:^|.*:)Version: (?P<version>.*)$") |
| if "python3-" in pn: |
| version_check_cmd = "find %s -name 'METADATA' | xargs grep '^Version: '" % pkgd |
| try: |
| output = subprocess.check_output(version_check_cmd, shell=True).decode("utf-8") |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("output:\n'''\n%s'''\n" % output) |
| possible_versions = [] |
| for line in output.split("\n"): |
| match = __regex_python_module_version__.match(line) |
| if match: |
| possible_versions.append(match.group("version")) |
| possible_versions = sorted(set(possible_versions)) |
| data_lines.append("possible versions: %s\n" % possible_versions) |
| if is_version_mismatch(possible_versions, pv): |
| data_lines.append("FINAL RESULT: MISMATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| bb.warn("Possible runtime versions %s do not match recipe version %s" % (possible_versions, pv)) |
| else: |
| data_lines.append("FINAL RESULT: MATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| got_quick_match_result = True |
| except: |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("result: RUN_FAILED\n\n") |
| if got_quick_match_result: |
| if enable_debug: |
| with open(debug_data_file, "w") as f: |
| f.writelines(data_lines) |
| return |
| |
| # handle .pc files |
| version_check_cmd = "find %s -name '*.pc' | xargs grep -i version" % pkgd |
| try: |
| output = subprocess.check_output(version_check_cmd, shell=True).decode("utf-8") |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("output:\n'''\n%s'''\n" % output) |
| possible_versions = get_possible_versions(output) |
| possible_versions = sorted(set(possible_versions)) |
| data_lines.append("possible versions: %s\n" % possible_versions) |
| if is_version_mismatch(possible_versions, pv): |
| if pn.startswith("lib"): |
| data_lines.append("FINAL RESULT: MISMATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| bb.warn("Possible runtime versions %s do not match recipe version %s" % (possible_versions, pv)) |
| got_quick_match_result = True |
| else: |
| data_lines.append("result: MISMATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| else: |
| data_lines.append("FINAL RESULT: MATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| got_quick_match_result = True |
| except: |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("result: RUN_FAILED\n\n") |
| if got_quick_match_result: |
| if enable_debug: |
| with open(debug_data_file, "w") as f: |
| f.writelines(data_lines) |
| return |
| |
| skipped_directories = [".debug", "ptest", "installed-tests", "tests", "test", "__pycache__", "testcases"] |
| # avoid checking configuration files, they don't give useful version information and some init scripts |
| # will kill all processes |
| skipped_directories.append("etc") |
| skipped_directories.append("go/src") |
| pkgd_libdir = pkgd + d.getVar("libdir") |
| pkgd_base_libdir = pkgd + d.getVar("base_libdir") |
| extra_exec_libdirs = [] |
| for root, dirs, files in os.walk(pkgd): |
| for dname in dirs: |
| fdir = os.path.join(root, dname) |
| if os.path.isdir(fdir) and fdir != pkgd_libdir and fdir != pkgd_base_libdir: |
| if fdir.startswith(pkgd_libdir) or fdir.startswith(pkgd_base_libdir): |
| for sd in skipped_directories: |
| if fdir.endswith("/" + sd) or ("/" + sd + "/") in fdir: |
| break |
| else: |
| extra_exec_libdirs.append(fdir) |
| for fname in files: |
| fpath = os.path.join(root, fname) |
| if os.path.isfile(fpath) and os.access(fpath, os.X_OK): |
| for sd in skipped_directories: |
| if ("/" + sd + "/") in fpath: |
| break |
| else: |
| if is_shared_library(fpath): |
| # we don't check shared libraries |
| continue |
| else: |
| executables.append(fpath) |
| if enable_debug: |
| data_lines.append("executables: %s\n" % executables) |
| |
| found_match = False |
| some_cmd_succeed = False |
| if not executables: |
| bb.debug(1, "No executable found for %s" % pn) |
| data_lines.append("FINAL RESULT: NO_EXECUTABLE_FOUND\n\n") |
| else: |
| # first we extend qemu_exec to include library path if needed |
| if extra_exec_libdirs: |
| qemu_exec += ":" + ":".join(extra_exec_libdirs) |
| orig_qemu_exec = qemu_exec |
| for fexec in executables: |
| qemu_exec = orig_qemu_exec |
| for version_option in ["--version", "-V", "-v", "--help"]: |
| if not is_elf_binary(fexec): |
| shebang = get_shebang(fexec) |
| interpreter = get_interpreter_from_shebang(shebang) |
| if not interpreter: |
| bb.debug(1, "file %s is not supported to run" % fexec) |
| elif interpreter.endswith("perl"): |
| perl5lib_extra = pkgd + d.getVar("libdir") + "/perl5/site_perl" |
| for p in glob.glob("%s/usr/share/*" % pkgd): |
| perl5lib_extra += ":%s" % p |
| qemu_exec += " -E PERL5LIB=%s:$PERL5LIB %s" % (perl5lib_extra, interpreter) |
| elif interpreter.endswith("python3"): |
| pythonpath_extra = glob.glob("%s%s/python3*/site-packages" % (pkgd, d.getVar("libdir"))) |
| if pythonpath_extra: |
| qemu_exec += " -E PYTHONPATH=%s:$PYTHONPATH %s" % (pythonpath_extra[0], interpreter) |
| else: |
| qemu_exec += " %s" % interpreter |
| # remove the '-E LD_LIBRARY_PATH=xxx' |
| qemu_exec = re.sub(r"-E\s+LD_LIBRARY_PATH=\S+", "", qemu_exec) |
| version_check_cmd_full = "%s %s %s" % (qemu_exec, fexec, version_option) |
| version_check_cmd = version_check_cmd_full |
| #version_check_cmd = "%s %s" % (os.path.relpath(fexec, pkgd), version_option) |
| |
| try: |
| cwd_temp = d.getVar("TMPDIR") + "/check-version-mismatch/cwd-temp/" + pn |
| os.makedirs(cwd_temp, exist_ok=True) |
| # avoid pseudo to manage any file we create |
| sp_env = os.environ.copy() |
| sp_env["PSEUDO_UNLOAD"] = "1" |
| output = subprocess.check_output(version_check_cmd_full, |
| shell=True, |
| stderr=subprocess.STDOUT, |
| cwd=cwd_temp, |
| timeout=10, |
| env=sp_env).decode("utf-8") |
| some_cmd_succeed = True |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("output:\n'''\n%s'''\n" % output) |
| if version_option == "--help": |
| max_lines = 5 |
| else: |
| max_lines = None |
| possible_versions = get_possible_versions(output, full_cmd=fexec, max_lines=max_lines) |
| if "." in pv: |
| possible_versions = [item for item in possible_versions if "." in item or item == "UNKNOWN"] |
| data_lines.append("possible versions: %s\n" % possible_versions) |
| if not possible_versions: |
| data_lines.append("result: NO_RUNTIME_VERSION_FOUND\n\n") |
| continue |
| possible_versions_all.extend(possible_versions) |
| possible_versions_all = sorted(set(possible_versions_all)) |
| if is_version_mismatch(possible_versions, pv): |
| data_lines.append("result: MISMATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| else: |
| found_match = True |
| data_lines.append("result: MATCH (%s v.s. %s)\n\n" % (possible_versions, pv)) |
| break |
| except: |
| data_lines.append("version_check_cmd: %s\n" % version_check_cmd) |
| data_lines.append("result: RUN_FAILED\n\n") |
| finally: |
| shutil.rmtree(cwd_temp) |
| if found_match: |
| break |
| if executables: |
| if found_match: |
| data_lines.append("FINAL RESULT: MATCH (%s v.s. %s)\n" % (possible_versions_all, pv)) |
| elif len(possible_versions_all) == 0: |
| if some_cmd_succeed: |
| bb.debug(1, "No valid runtime version found") |
| data_lines.append("FINAL RESULT: NO_VALID_RUNTIME_VERSION_FOUND\n") |
| else: |
| bb.debug(1, "All version check command failed") |
| data_lines.append("FINAL RESULT: RUN_FAILED\n") |
| else: |
| bb.warn("Possible runtime versions %s do not match recipe version %s" % (possible_versions_all, pv)) |
| data_lines.append("FINAL RESULT: MISMATCH (%s v.s. %s)\n" % (possible_versions_all, pv)) |
| |
| if enable_debug: |
| with open(debug_data_file, "w") as f: |
| f.writelines(data_lines) |
| |
| # clean up stale processes |
| process_name_common_prefix = "%s %s" % (' '.join(qemu_exec.split()[1:]), pkgd) |
| find_stale_process_cmd = "ps -e -o pid,args | grep -v grep | grep -F '%s'" % process_name_common_prefix |
| try: |
| stale_process_output = subprocess.check_output(find_stale_process_cmd, shell=True).decode("utf-8") |
| stale_process_pids = [] |
| for line in stale_process_output.split("\n"): |
| line = line.strip() |
| if not line: |
| continue |
| pid = line.split()[0] |
| stale_process_pids.append(pid) |
| for pid in stale_process_pids: |
| os.kill(int(pid), signal.SIGKILL) |
| except Exception as e: |
| bb.debug(1, "No stale process") |
| } |
| |
| addtask do_package_check_version_mismatch after do_prepare_recipe_sysroot do_package before do_build |
| |
| do_build[rdeptask] += "do_package_check_version_mismatch" |
| do_rootfs[recrdeptask] += "do_package_check_version_mismatch" |
| |
| SSTATETASKS += "do_package_check_version_mismatch" |
| do_package_check_version_mismatch[sstate-inputdirs] = "" |
| do_package_check_version_mismatch[sstate-outputdirs] = "" |
| python do_package_check_version_mismatch_setscene () { |
| sstate_setscene(d) |
| } |
| addtask do_package_check_version_mismatch_setscene |
