| # |
| # Copyright OpenEmbedded Contributors |
| # |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| """Code for parsing OpenEmbedded license strings""" |
| |
| import ast |
| import re |
| import oe.qa |
| from fnmatch import fnmatchcase as fnmatch |
| |
| def license_ok(license, dont_want_licenses): |
| """ Return False if License exist in dont_want_licenses else True """ |
| for dwl in dont_want_licenses: |
| if fnmatch(license, dwl): |
| return False |
| return True |
| |
| def obsolete_license_list(): |
| return ["AGPL-3", "AGPL-3+", "AGPLv3", "AGPLv3+", "AGPLv3.0", "AGPLv3.0+", "AGPL-3.0", "AGPL-3.0+", "BSD-0-Clause", |
| "GPL-1", "GPL-1+", "GPLv1", "GPLv1+", "GPLv1.0", "GPLv1.0+", "GPL-1.0", "GPL-1.0+", "GPL-2", "GPL-2+", "GPLv2", |
| "GPLv2+", "GPLv2.0", "GPLv2.0+", "GPL-2.0", "GPL-2.0+", "GPL-3", "GPL-3+", "GPLv3", "GPLv3+", "GPLv3.0", "GPLv3.0+", |
| "GPL-3.0", "GPL-3.0+", "LGPLv2", "LGPLv2+", "LGPLv2.0", "LGPLv2.0+", "LGPL-2.0", "LGPL-2.0+", "LGPL2.1", "LGPL2.1+", |
| "LGPLv2.1", "LGPLv2.1+", "LGPL-2.1", "LGPL-2.1+", "LGPLv3", "LGPLv3+", "LGPL-3.0", "LGPL-3.0+", "MPL-1", "MPLv1", |
| "MPLv1.1", "MPLv2", "MIT-X", "MIT-style", "openssl", "PSF", "PSFv2", "Python-2", "Apachev2", "Apache-2", "Artisticv1", |
| "Artistic-1", "AFL-2", "AFL-1", "AFLv2", "AFLv1", "CDDLv1", "CDDL-1", "EPLv1.0", "FreeType", "Nauman", |
| "tcl", "vim", "SGIv1"] |
| |
| class LicenseError(Exception): |
| pass |
| |
| class LicenseSyntaxError(LicenseError): |
| def __init__(self, licensestr, exc): |
| self.licensestr = licensestr |
| self.exc = exc |
| LicenseError.__init__(self) |
| |
| def __str__(self): |
| return "error in '%s': %s" % (self.licensestr, self.exc) |
| |
| class InvalidLicense(LicenseError): |
| def __init__(self, license): |
| self.license = license |
| LicenseError.__init__(self) |
| |
| def __str__(self): |
| return "invalid characters in license '%s'" % self.license |
| |
| license_operator_chars = '&|() ' |
| license_operator = re.compile(r'([' + license_operator_chars + '])') |
| license_pattern = re.compile(r'[a-zA-Z0-9.+_\-]+$') |
| |
| class LicenseVisitor(ast.NodeVisitor): |
| """Get elements based on OpenEmbedded license strings""" |
| def get_elements(self, licensestr): |
| new_elements = [] |
| elements = list([x for x in license_operator.split(licensestr) if x.strip()]) |
| for pos, element in enumerate(elements): |
| if license_pattern.match(element): |
| if pos > 0 and license_pattern.match(elements[pos-1]): |
| new_elements.append('&') |
| element = '"' + element + '"' |
| elif not license_operator.match(element): |
| raise InvalidLicense(element) |
| new_elements.append(element) |
| |
| return new_elements |
| |
| """Syntax tree visitor which can accept elements previously generated with |
| OpenEmbedded license string""" |
| def visit_elements(self, elements): |
| self.visit(ast.parse(' '.join(elements))) |
| |
| """Syntax tree visitor which can accept OpenEmbedded license strings""" |
| def visit_string(self, licensestr): |
| self.visit_elements(self.get_elements(licensestr)) |
| |
| class FlattenVisitor(LicenseVisitor): |
| """Flatten a license tree (parsed from a string) by selecting one of each |
| set of OR options, in the way the user specifies""" |
| def __init__(self, choose_licenses): |
| self.choose_licenses = choose_licenses |
| self.licenses = [] |
| LicenseVisitor.__init__(self) |
| |
| def visit_Constant(self, node): |
| self.licenses.append(node.value) |
| |
| def visit_BinOp(self, node): |
| if isinstance(node.op, ast.BitOr): |
| left = FlattenVisitor(self.choose_licenses) |
| left.visit(node.left) |
| |
| right = FlattenVisitor(self.choose_licenses) |
| right.visit(node.right) |
| |
| selected = self.choose_licenses(left.licenses, right.licenses) |
| self.licenses.extend(selected) |
| else: |
| self.generic_visit(node) |
| |
| def flattened_licenses(licensestr, choose_licenses): |
| """Given a license string and choose_licenses function, return a flat list of licenses""" |
| flatten = FlattenVisitor(choose_licenses) |
| try: |
| flatten.visit_string(licensestr) |
| except SyntaxError as exc: |
| raise LicenseSyntaxError(licensestr, exc) |
| return flatten.licenses |
| |
| def is_included(licensestr, include_licenses=None, exclude_licenses=None): |
| """Given a license string, a list of licenses to include and a list of |
| licenses to exclude, determine if the license string matches the include |
| list and does not match the exclude list. |
| |
| Returns a tuple holding the boolean state and a list of the applicable |
| licenses that were excluded if state is False, or the licenses that were |
| included if the state is True.""" |
| |
| def include_license(license): |
| return any(fnmatch(license, pattern) for pattern in include_licenses) |
| |
| def exclude_license(license): |
| return any(fnmatch(license, pattern) for pattern in exclude_licenses) |
| |
| def choose_licenses(alpha, beta): |
| """Select the option in an OR which is the 'best' (has the most |
| included licenses and no excluded licenses).""" |
| # The factor 1000 below is arbitrary, just expected to be much larger |
| # than the number of licenses actually specified. That way the weight |
| # will be negative if the list of licenses contains an excluded license, |
| # but still gives a higher weight to the list with the most included |
| # licenses. |
| alpha_weight = (len(list(filter(include_license, alpha))) - |
| 1000 * (len(list(filter(exclude_license, alpha))) > 0)) |
| beta_weight = (len(list(filter(include_license, beta))) - |
| 1000 * (len(list(filter(exclude_license, beta))) > 0)) |
| if alpha_weight >= beta_weight: |
| return alpha |
| else: |
| return beta |
| |
| if not include_licenses: |
| include_licenses = ['*'] |
| |
| if not exclude_licenses: |
| exclude_licenses = [] |
| |
| licenses = flattened_licenses(licensestr, choose_licenses) |
| excluded = [lic for lic in licenses if exclude_license(lic)] |
| included = [lic for lic in licenses if include_license(lic)] |
| if excluded: |
| return False, excluded |
| else: |
| return True, included |
| |
| class ManifestVisitor(LicenseVisitor): |
| """Walk license tree (parsed from a string) removing the incompatible |
| licenses specified""" |
| def __init__(self, dont_want_licenses, canonical_license, d): |
| self._dont_want_licenses = dont_want_licenses |
| self._canonical_license = canonical_license |
| self._d = d |
| self._operators = [] |
| |
| self.licenses = [] |
| self.licensestr = '' |
| |
| LicenseVisitor.__init__(self) |
| |
| def visit(self, node): |
| if isinstance(node, ast.Constant): |
| lic = node.value |
| |
| if license_ok(self._canonical_license(self._d, lic), |
| self._dont_want_licenses) == True: |
| if self._operators: |
| ops = [] |
| for op in self._operators: |
| if op == '[': |
| ops.append(op) |
| elif op == ']': |
| ops.append(op) |
| else: |
| if not ops: |
| ops.append(op) |
| elif ops[-1] in ['[', ']']: |
| ops.append(op) |
| else: |
| ops[-1] = op |
| |
| for op in ops: |
| if op == '[' or op == ']': |
| self.licensestr += op |
| elif self.licenses: |
| self.licensestr += ' ' + op + ' ' |
| |
| self._operators = [] |
| |
| self.licensestr += lic |
| self.licenses.append(lic) |
| elif isinstance(node, ast.BitAnd): |
| self._operators.append("&") |
| elif isinstance(node, ast.BitOr): |
| self._operators.append("|") |
| elif isinstance(node, ast.List): |
| self._operators.append("[") |
| elif isinstance(node, ast.Load): |
| self.licensestr += "]" |
| |
| self.generic_visit(node) |
| |
| def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): |
| """Given a license string and dont_want_licenses list, |
| return license string filtered and a list of licenses""" |
| manifest = ManifestVisitor(dont_want_licenses, canonical_license, d) |
| |
| try: |
| elements = manifest.get_elements(licensestr) |
| |
| # Replace '()' to '[]' for handle in ast as List and Load types. |
| elements = ['[' if e == '(' else e for e in elements] |
| elements = [']' if e == ')' else e for e in elements] |
| |
| manifest.visit_elements(elements) |
| except SyntaxError as exc: |
| raise LicenseSyntaxError(licensestr, exc) |
| |
| # Replace '[]' to '()' for output correct license. |
| manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') |
| |
| return (manifest.licensestr, manifest.licenses) |
| |
| class ListVisitor(LicenseVisitor): |
| """Record all different licenses found in the license string""" |
| def __init__(self): |
| self.licenses = set() |
| |
| def visit_Constant(self, node): |
| self.licenses.add(node.value) |
| |
| def list_licenses(licensestr): |
| """Simply get a list of all licenses mentioned in a license string. |
| Binary operators are not applied or taken into account in any way""" |
| visitor = ListVisitor() |
| try: |
| visitor.visit_string(licensestr) |
| except SyntaxError as exc: |
| raise LicenseSyntaxError(licensestr, exc) |
| return visitor.licenses |
| |
| def apply_pkg_license_exception(pkg, bad_licenses, exceptions): |
| """Return remaining bad licenses after removing any package exceptions""" |
| |
| return [lic for lic in bad_licenses if pkg + ':' + lic not in exceptions] |
| |
| def return_spdx(d, license): |
| """ |
| This function returns the spdx mapping of a license if it exists. |
| """ |
| return d.getVarFlag('SPDXLICENSEMAP', license) |
| |
| def canonical_license(d, license): |
| """ |
| Return the canonical (SPDX) form of the license if available (so GPLv3 |
| becomes GPL-3.0-only) or the passed license if there is no canonical form. |
| """ |
| return d.getVarFlag('SPDXLICENSEMAP', license) or license |
| |
| def expand_wildcard_licenses(d, wildcard_licenses): |
| """ |
| There are some common wildcard values users may want to use. Support them |
| here. |
| """ |
| licenses = set(wildcard_licenses) |
| mapping = { |
| "AGPL-3.0*" : ["AGPL-3.0-only", "AGPL-3.0-or-later"], |
| "GPL-3.0*" : ["GPL-3.0-only", "GPL-3.0-or-later"], |
| "LGPL-3.0*" : ["LGPL-3.0-only", "LGPL-3.0-or-later"], |
| } |
| for k in mapping: |
| if k in wildcard_licenses: |
| licenses.remove(k) |
| for item in mapping[k]: |
| licenses.add(item) |
| |
| for l in licenses: |
| if l in obsolete_license_list(): |
| bb.fatal("Error, %s is an obsolete license, please use an SPDX reference in INCOMPATIBLE_LICENSE" % l) |
| if "*" in l: |
| bb.fatal("Error, %s is an invalid license wildcard entry" % l) |
| |
| return list(licenses) |
| |
| def incompatible_license_contains(license, truevalue, falsevalue, d): |
| license = canonical_license(d, license) |
| bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() |
| bad_licenses = expand_wildcard_licenses(d, bad_licenses) |
| return truevalue if license in bad_licenses else falsevalue |
| |
| def incompatible_pkg_license(d, dont_want_licenses, license): |
| # Handles an "or" or two license sets provided by |
| # flattened_licenses(), pick one that works if possible. |
| def choose_lic_set(a, b): |
| return a if all(license_ok(canonical_license(d, lic), |
| dont_want_licenses) for lic in a) else b |
| |
| try: |
| licenses = flattened_licenses(license, choose_lic_set) |
| except LicenseError as exc: |
| bb.fatal('%s: %s' % (d.getVar('P'), exc)) |
| |
| incompatible_lic = [] |
| for l in licenses: |
| license = canonical_license(d, l) |
| if not license_ok(license, dont_want_licenses): |
| incompatible_lic.append(license) |
| |
| return sorted(incompatible_lic) |
| |
| def incompatible_license(d, dont_want_licenses, package=None): |
| """ |
| This function checks if a recipe has only incompatible licenses. It also |
| take into consideration 'or' operand. dont_want_licenses should be passed |
| as canonical (SPDX) names. |
| """ |
| license = d.getVar("LICENSE:%s" % package) if package else None |
| if not license: |
| license = d.getVar('LICENSE') |
| |
| return incompatible_pkg_license(d, dont_want_licenses, license) |
| |
| def check_license_flags(d): |
| """ |
| This function checks if a recipe has any LICENSE_FLAGS that |
| aren't acceptable. |
| |
| If it does, it returns the all LICENSE_FLAGS missing from the list |
| of acceptable license flags, or all of the LICENSE_FLAGS if there |
| is no list of acceptable flags. |
| |
| If everything is is acceptable, it returns None. |
| """ |
| |
| def license_flag_matches(flag, acceptlist, pn): |
| """ |
| Return True if flag matches something in acceptlist, None if not. |
| |
| Before we test a flag against the acceptlist, we append _${PN} |
| to it. We then try to match that string against the |
| acceptlist. This covers the normal case, where we expect |
| LICENSE_FLAGS to be a simple string like 'commercial', which |
| the user typically matches exactly in the acceptlist by |
| explicitly appending the package name e.g 'commercial_foo'. |
| If we fail the match however, we then split the flag across |
| '_' and append each fragment and test until we either match or |
| run out of fragments. |
| """ |
| flag_pn = ("%s_%s" % (flag, pn)) |
| for candidate in acceptlist: |
| if flag_pn == candidate: |
| return True |
| |
| flag_cur = "" |
| flagments = flag_pn.split("_") |
| flagments.pop() # we've already tested the full string |
| for flagment in flagments: |
| if flag_cur: |
| flag_cur += "_" |
| flag_cur += flagment |
| for candidate in acceptlist: |
| if flag_cur == candidate: |
| return True |
| return False |
| |
| def all_license_flags_match(license_flags, acceptlist): |
| """ Return all unmatched flags, None if all flags match """ |
| pn = d.getVar('PN') |
| split_acceptlist = acceptlist.split() |
| flags = [] |
| for flag in license_flags.split(): |
| if not license_flag_matches(flag, split_acceptlist, pn): |
| flags.append(flag) |
| return flags if flags else None |
| |
| license_flags = d.getVar('LICENSE_FLAGS') |
| if license_flags: |
| acceptlist = d.getVar('LICENSE_FLAGS_ACCEPTED') |
| if not acceptlist: |
| return license_flags.split() |
| unmatched_flags = all_license_flags_match(license_flags, acceptlist) |
| if unmatched_flags: |
| return unmatched_flags |
| return None |
| |
| def check_license_format(d): |
| """ |
| This function checks if LICENSE is well defined, |
| Validate operators in LICENSES. |
| No spaces are allowed between LICENSES. |
| """ |
| pn = d.getVar('PN') |
| licenses = d.getVar('LICENSE') |
| |
| elements = list(filter(lambda x: x.strip(), license_operator.split(licenses))) |
| for pos, element in enumerate(elements): |
| if license_pattern.match(element): |
| if pos > 0 and license_pattern.match(elements[pos - 1]): |
| oe.qa.handle_error('license-format', |
| '%s: LICENSE value "%s" has an invalid format - license names ' \ |
| 'must be separated by the following characters to indicate ' \ |
| 'the license selection: %s' % |
| (pn, licenses, license_operator_chars), d) |
| elif not license_operator.match(element): |
| oe.qa.handle_error('license-format', |
| '%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \ |
| 'in the valid list of separators (%s)' % |
| (pn, licenses, element, license_operator_chars), d) |
| |
| def skip_incompatible_package_licenses(d, pkgs): |
| if not pkgs: |
| return {} |
| |
| pn = d.getVar("PN") |
| |
| check_license = False if pn.startswith("nativesdk-") else True |
| for t in ["-native", "-cross-${TARGET_ARCH}", "-cross-initial-${TARGET_ARCH}", |
| "-crosssdk-${SDK_SYS}", "-crosssdk-initial-${SDK_SYS}", |
| "-cross-canadian-${TRANSLATED_TARGET_ARCH}"]: |
| if pn.endswith(d.expand(t)): |
| check_license = False |
| if pn.startswith("gcc-source-"): |
| check_license = False |
| |
| bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() |
| if not check_license or not bad_licenses: |
| return {} |
| |
| bad_licenses = expand_wildcard_licenses(d, bad_licenses) |
| |
| exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() |
| |
| for lic_exception in exceptions: |
| if ":" in lic_exception: |
| lic_exception = lic_exception.split(":")[1] |
| if lic_exception in obsolete_license_list(): |
| bb.fatal("Obsolete license %s used in INCOMPATIBLE_LICENSE_EXCEPTIONS" % lic_exception) |
| |
| skipped_pkgs = {} |
| for pkg in pkgs: |
| remaining_bad_licenses = apply_pkg_license_exception(pkg, bad_licenses, exceptions) |
| |
| incompatible_lic = incompatible_license(d, remaining_bad_licenses, pkg) |
| if incompatible_lic: |
| skipped_pkgs[pkg] = incompatible_lic |
| |
| return skipped_pkgs |
| |
| def tidy_licenses(value): |
| """ |
| Flat, split and sort licenses. |
| """ |
| from oe.license import flattened_licenses |
| |
| def _choose(a, b): |
| str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold) |
| return ["(%s | %s)" % (str_a, str_b)] |
| |
| if not isinstance(value, str): |
| value = " & ".join(value) |
| |
| return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold) |
