From 05d9197eb7ddda08e0f9cf9357b9a875d4475644 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Tue, 14 Jun 2022 17:59:44 -0400 Subject: [PATCH 01/43] Add support for external licenses in scans #480 This adds `-dir` or `--additional-directories` as a command line option in license detection. This allows users to specify paths to directories of licenses and rules they'd like to use during license detection, but would not like to add to the ScanCode database of licenses. This involves adding a new option in `licensedcode/plugin_license.py`, and this option is used as a parameter in `scancode/api.py`. In this approach, the licenses and rules contained in these additional directories are combined with the existing licenses and rules in the ScanCode database to produce a single index. The code for this is found in `licensedcode/cache.py` and the helper methods for loading these licenses and rules are found in `licensedcode/models.py`. This commit also includes a unit test to verify that license detection succeeds with an additional directory found in `tests/licensedcode/test_plugin_license.py`. Part of the setup for the unit test and future tests involves creating a new directory in `tests/licensedcode/data` that contains sample external licenses used in the unit tests. Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 103 +++++++++++++-- src/licensedcode/models.py | 63 +++++++++ src/licensedcode/plugin_license.py | 15 ++- src/scancode/api.py | 4 +- .../example1/licenses/example1.LICENSE | 1 + .../example1/licenses/example1.yml | 5 + .../example1/rules/example1_1.RULE | 2 + .../example1/rules/example1_1.yml | 2 + .../example2/licenses/example2.LICENSE | 7 + .../example2/licenses/example2.yml | 5 + .../example2/rules/example2.RULE | 1 + .../example2/rules/example2.yml | 2 + .../external_licenses/scan.expected.json | 81 ++++++++++++ .../external_licenses/scan/license.txt | 6 + .../scan_multiple.expected.json | 120 ++++++++++++++++++ tests/licensedcode/test_plugin_license.py | 38 ++++++ 16 files changed, 444 insertions(+), 11 deletions(-) create mode 100644 tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE create mode 100644 tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml create mode 100644 tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE create mode 100644 tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml create mode 100644 tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE create mode 100644 tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml create mode 100644 tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE create mode 100644 tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml create mode 100644 tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json create mode 100644 tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt create mode 100644 tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index cb3c788be6a..feabad893b3 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -35,6 +35,7 @@ LICENSE_INDEX_FILENAME = 'index_cache' LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile' LICENSE_CHECKSUM_FILE = 'scancode_license_index_tree_checksums' +CACHED_DIRECTORIES_FILENAME = 'cached_directories' @attr.s(slots=True) @@ -58,6 +59,7 @@ def load_or_build( timeout=LICENSE_INDEX_LOCK_TIMEOUT, licenses_data_dir=None, rules_data_dir=None, + additional_directories=None, ): """ Load or build and save and return a LicenseCache object. @@ -66,7 +68,8 @@ def load_or_build( On the side, we load cached or build license db, SPDX symbols and other license-related data structures. - - If the cache exists, it is returned unless corrupted or ``force`` is True. + - If the cache exists, it is returned unless corrupted, ``force`` is True, or if we pass in additional + directories containing licenses that are not present in the existing cache. - If the cache does not exist, a new index is built and cached. - If ``index_all_languages`` is True, include texts in all languages when building the license index. Otherwise, only include the English license \ @@ -75,12 +78,17 @@ def load_or_build( idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) create_dir(idx_cache_dir) cache_file = os.path.join(idx_cache_dir, LICENSE_INDEX_FILENAME) + cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) has_cache = os.path.exists(cache_file) and os.path.getsize(cache_file) # bypass build if cache exists if has_cache and not force: try: + # save the list of additional directories included in the cache, or None if the cache does not + # include any additional directories + with open(cached_directories_file, 'wb') as file: + pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) return load_cache_file(cache_file) except Exception as e: # work around some rare Windows quirks @@ -92,6 +100,8 @@ def load_or_build( from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd from licensedcode.models import load_licenses + from licensedcode.models import load_licenses_from_multiple_dirs + from licensedcode.models import get_license_dirs from scancode import lockfile licenses_data_dir = licenses_data_dir or ldd @@ -106,13 +116,21 @@ def load_or_build( # Here, the cache is either stale or non-existing: we need to # rebuild all cached data (e.g. mostly the index) and cache it - licenses_db = load_licenses(licenses_data_dir=licenses_data_dir) + if additional_directories: + additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) + combined_directories = [licenses_data_dir] + additional_license_dirs + licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories) + else: + licenses_db = load_licenses(licenses_data_dir=licenses_data_dir) + # create a single merged index containing license data from licenses_data_dir + # and data from additional directories index = build_index( licenses_db=licenses_db, licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir, index_all_languages=index_all_languages, + additional_directories=additional_directories, ) spdx_symbols = build_spdx_symbols(licenses_db=licenses_db) @@ -131,6 +149,11 @@ def load_or_build( with open(cache_file, 'wb') as fn: pickle.dump(license_cache, fn, protocol=PICKLE_PROTOCOL) + # save the list of additional directories included in the cache, or None if the cache does not + # include any additional directories + with open(cached_directories_file, 'wb') as file: + pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) + return license_cache except lockfile.LockTimeout: @@ -143,27 +166,50 @@ def build_index( licenses_data_dir=None, rules_data_dir=None, index_all_languages=False, + additional_directories=None, ): """ Return an index built from rules and licenses directories If ``index_all_languages`` is True, include texts and rules in all languages. Otherwise, only include the English license texts and rules (the default) + If ``additional_directories`` is not None, we will include licenses and rules + from these additional directories in the returned index. """ from licensedcode.index import LicenseIndex + from licensedcode.models import get_license_dirs + from licensedcode.models import get_rule_dirs from licensedcode.models import get_rules + from licensedcode.models import get_rules_from_multiple_dirs from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import get_license_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd from licensedcode.models import load_licenses + from licensedcode.models import load_licenses_from_multiple_dirs from licensedcode.legalese import common_license_words licenses_data_dir = licenses_data_dir or ldd rules_data_dir = rules_data_dir or rdd - licenses_db = licenses_db or load_licenses(licenses_data_dir=licenses_data_dir) - rules = get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) + if not licenses_db: + if additional_directories: + # combine the licenses in these additional directories with the licenses in the original DB + additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) + combined_license_directories = [licenses_data_dir] + additional_license_dirs + # generate a single combined license db with all licenses + licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories) + else: + licenses_db = load_licenses(licenses_data_dir=licenses_data_dir) + + if additional_directories: + # if we have additional directories, extract the rules from them + additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) + # then combine the rules in these additional directories with the rules in the original rules directory + combined_rule_directories = [rules_data_dir] + additional_rule_dirs + rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories) + else: + rules = get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) legalese = common_license_words spdx_tokens = set(get_all_spdx_key_tokens(licenses_db)) @@ -299,7 +345,7 @@ def build_unknown_spdx_symbol(licenses_db=None): return LicenseSymbolLike(licenses_db['unknown-spdx']) -def get_cache(force=False, index_all_languages=False): +def get_cache(force=False, index_all_languages=False, additional_directories=None): """ Return a LicenseCache either rebuilt, cached or loaded from disk. @@ -307,16 +353,18 @@ def get_cache(force=False, index_all_languages=False): building the license index. Otherwise, only include the English license \ texts and rules (the default) """ - populate_cache(force=force, index_all_languages=index_all_languages) + populate_cache(force=force, index_all_languages=index_all_languages, additional_directories=additional_directories) global _LICENSE_CACHE return _LICENSE_CACHE -def populate_cache(force=False, index_all_languages=False): +def populate_cache(force=False, index_all_languages=False, additional_directories=None): """ Load or build and cache a LicenseCache. Return None. """ global _LICENSE_CACHE + if need_cache_rebuild(additional_directories): + force = True if force or not _LICENSE_CACHE: _LICENSE_CACHE = LicenseCache.load_or_build( licensedcode_cache_dir=licensedcode_cache_dir, @@ -325,9 +373,42 @@ def populate_cache(force=False, index_all_languages=False): index_all_languages=index_all_languages, # used for testing only timeout=LICENSE_INDEX_LOCK_TIMEOUT, + additional_directories=additional_directories, ) +def need_cache_rebuild(additional_directories): + """ + Return true if we need to rebuild the index cache. + """ + idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) + cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) + + has_cached_directories = os.path.exists(cached_directories_file) + should_rebuild_cache = False + + if has_cached_directories: + # if we have cached additional directories of licenses, check if those licenses are equal to the additional + # directories passed in + with open(cached_directories_file, 'rb') as file: + # it's possible that pickle.load(file) results in None + try: + cached_additional_directories = pickle.load(file) + except EOFError: + cached_additional_directories = set() + + # we need to rebuild the cache if the list of additional directories we passed in is not a subset of + # the set of additional directories currently included in the index cache + should_rebuild_cache = additional_directories is not None \ + and not set(additional_directories).issubset(cached_additional_directories) + else: + # otherwise, we don't have a file of cached directories. If there are additional directories passed in, + # we know we need to make a new cache file. + if additional_directories: + should_rebuild_cache = True + return should_rebuild_cache + + def load_cache_file(cache_file): """ Return a LicenseCache loaded from ``cache_file``. @@ -346,11 +427,15 @@ def load_cache_file(cache_file): raise Exception(msg) from e -def get_index(force=False, index_all_languages=False): +def get_index(force=False, index_all_languages=False, additional_directories=None): """ Return and eventually build and cache a LicenseIndex. """ - return get_cache(force=force, index_all_languages=index_all_languages).index + return get_cache( + force=force, + index_all_languages=index_all_languages, + additional_directories=additional_directories + ).index get_cached_index = get_index diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index 321ae77525c..a4efef20815 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -22,6 +22,7 @@ from os.path import exists from os.path import join from time import time +from pathlib import Path import attr import saneyaml @@ -770,6 +771,68 @@ def get_rules( return chain(licenses_as_rules, rules) +def get_license_dirs( + additional_dirs, +): + """ + Takes in a list of additional directories specified during license detection + and produces a list of all the subdirectories containing license files. + """ + # convert to absolute path in case user passes in a relative path, which messes up building rules from licenses + return [f"{str(Path(path).absolute())}/licenses" for path in additional_dirs] + + +def get_rule_dirs( + additional_dirs, +): + """ + Takes in a list of additional directories specified during license detection + and produces a list of all the subdirectories containing rule files. + """ + return [f"{str(Path(path).absolute())}/rules" for path in additional_dirs] + + +def load_licenses_from_multiple_dirs( + license_directories, + with_deprecated=False, +): + """ + Takes in a list of directories containing additional licenses to use in + license detection and combines all the licenses into the same mapping. + """ + combined_licenses = {} + for license_dir in license_directories: + licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False) + # this syntax for merging is described here: https://stackoverflow.com/a/26853961 + combined_licenses = {**combined_licenses, **licenses} + return combined_licenses + + +def get_rules_from_multiple_dirs( + licenses_db, + rule_directories, +): + """ + Takes in a license database, which is a mapping from key->License objects, + and a list of all directories containing rules to use in license detection. + Combines all rules together into the same data structure and validates them. + """ + if rule_directories: + combined_rules = [] + for rules_dir in rule_directories: + r = list(load_rules( + rules_data_dir=rules_dir, + )) + combined_rules.append(r) + # flatten lists of rules into a single iterable + rules = list(chain.from_iterable(combined_rules)) + validate_rules(rules, licenses_db) + licenses_as_rules = build_rules_from_licenses(licenses_db) + return chain(licenses_as_rules, rules) + else: + return get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) + + class InvalidRule(Exception): pass diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 8658ffbc356..ff1ba3fc6d8 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -18,6 +18,7 @@ from commoncode.resource import clean_path from plugincode.scan import ScanPlugin from plugincode.scan import scan_impl +import click from scancode.api import SCANCODE_LICENSEDB_URL @@ -139,6 +140,15 @@ class LicenseScanner(ScanPlugin): help_group=SCAN_OPTIONS_GROUP, ), + PluggableCommandLineOption( + ('-dir', '--additional_directories'), + required_options=['license'], + multiple=True, + type=click.Path(exists=True, readable=True, path_type=str), + help='Include additional directories for license detection.', + help_group=SCAN_OPTIONS_GROUP, + ), + PluggableCommandLineOption( ('--reindex-licenses',), is_flag=True, is_eager=True, @@ -167,7 +177,8 @@ def setup(self, **kwargs): loaded index. """ from licensedcode.cache import populate_cache - populate_cache() + additional_directories = kwargs.get('additional_directories') + populate_cache(additional_directories=additional_directories) def get_scanner( self, @@ -176,6 +187,7 @@ def get_scanner( license_text_diagnostics=False, license_url_template=SCANCODE_LICENSEDB_URL, unknown_licenses=False, + additional_directories=None, **kwargs ): @@ -186,6 +198,7 @@ def get_scanner( license_text_diagnostics=license_text_diagnostics, license_url_template=license_url_template, unknown_licenses=unknown_licenses, + additional_directories=additional_directories, ) def process_codebase(self, codebase, unknown_licenses, **kwargs): diff --git a/src/scancode/api.py b/src/scancode/api.py index 1f3333c6a4b..1acd26ef174 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -142,6 +142,7 @@ def get_licenses( license_url_template=SCANCODE_LICENSEDB_URL, unknown_licenses=False, deadline=sys.maxsize, + additional_directories=None, **kwargs, ): """ @@ -168,7 +169,7 @@ def get_licenses( from licensedcode import cache from licensedcode.spans import Span - idx = cache.get_index() + idx = cache.get_index(additional_directories=additional_directories) detected_licenses = [] detected_expressions = [] @@ -252,6 +253,7 @@ def _licenses_data_from_match( result['homepage_url'] = lic.homepage_url result['text_url'] = lic.text_urls[0] if lic.text_urls else '' result['reference_url'] = license_url_template.format(lic.key) + # TODO: change this in the case of a private license? result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key) result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key) diff --git a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE new file mode 100644 index 00000000000..8fe2a4b5ad1 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE @@ -0,0 +1 @@ +The quick brown fox jumps over the lazy dog. \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml new file mode 100644 index 00000000000..d7d1ea640ec --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml @@ -0,0 +1,5 @@ +key: example1 +short_name: Example External License 1 +name: Example External License 1 +category: Permissive +owner: NexB \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE b/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE new file mode 100644 index 00000000000..ef512c3e024 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE @@ -0,0 +1,2 @@ +The quick brown fox jumps over the lazy dog. +The quick brown fox jumps over the lazy dog. \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml b/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml new file mode 100644 index 00000000000..96535e6c24b --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml @@ -0,0 +1,2 @@ +license_expression: example1 +is_license_text: yes \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE new file mode 100644 index 00000000000..4abca0c149f --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE @@ -0,0 +1,7 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi +ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit +in voluptate velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia +deserunt mollit anim id est laborum. diff --git a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml new file mode 100644 index 00000000000..d255bd8d70e --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml @@ -0,0 +1,5 @@ +key: example2 +short_name: Example External License 2 +name: Example External License 2 +category: Permissive +owner: NexB \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE b/tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE new file mode 100644 index 00000000000..c27a1f5e008 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit diff --git a/tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml b/tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml new file mode 100644 index 00000000000..0a6aebb4284 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml @@ -0,0 +1,2 @@ +license_expression: example2 +is_license_text: yes \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json new file mode 100644 index 00000000000..cc9b8a31704 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -0,0 +1,81 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "-dir": "", + "--json": "", + "--license": true, + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "output_format_version": "2.0.0", + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "linux", + "cpu_architecture": "64", + "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", + "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", + "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + }, + "spdx_license_list_version": "3.16", + "files_count": 2 + } + } + ], + "files": [ + { + "path": "license.txt", + "type": "file", + "licenses": [ + { + "key": "example1", + "score": 100.0, + "name": "Example External License 1", + "short_name": "Example External License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "spdx_license_key": "scancode-example1", + "spdx_url": "https://spdx.org/licenses/scancode-example1", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example1.LICENSE", + "license_expression": "example1", + "licenses": [ + "example1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 9, + "matched_length": 9, + "match_coverage": 100.0, + "rule_relevance": 100 + } + } + ], + "license_expressions": [ + "example1" + ], + "percentage_of_license_text": 10.98, + "scan_errors": [] + } + ] +} diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt b/tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt new file mode 100644 index 00000000000..0eacf6b73a6 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt @@ -0,0 +1,6 @@ +The quick brown fox jumps over the lazy dog. Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud +exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit +in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +This is a test license. diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json new file mode 100644 index 00000000000..e0128e97e5d --- /dev/null +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -0,0 +1,120 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "-dir": "", + "--json": "", + "--license": true, + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "output_format_version": "2.0.0", + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "linux", + "cpu_architecture": "64", + "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", + "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", + "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + }, + "spdx_license_list_version": "3.16", + "files_count": 2 + } + } + ], + "files": [ + { + "path": "license.txt", + "type": "file", + "licenses": [ + { + "key": "example1", + "score": 100.0, + "name": "Example External License 1", + "short_name": "Example External License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "spdx_license_key": "scancode-example1", + "spdx_url": "https://spdx.org/licenses/scancode-example1", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example1.LICENSE", + "license_expression": "example1", + "licenses": [ + "example1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 9, + "matched_length": 9, + "match_coverage": 100.0, + "rule_relevance": 100 + } + }, + { + "key": "example2", + "score": 100.0, + "name": "Example External License 2", + "short_name": "Example External License 2", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example2", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example2.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example2.yml", + "spdx_license_key": "scancode-example2", + "spdx_url": "https://spdx.org/licenses/scancode-example2", + "start_line": 1, + "end_line": 5, + "matched_rule": { + "identifier": "example2.LICENSE", + "license_expression": "example2", + "licenses": [ + "example2" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 69, + "matched_length": 69, + "match_coverage": 100.0, + "rule_relevance": 100 + } + } + ], + "license_expressions": [ + "example1", + "example2" + ], + "percentage_of_license_text": 95.12, + "scan_errors": [] + } + ] +} diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 0d5d20a469a..5eada5c99d9 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -242,6 +242,44 @@ def test_reindex_licenses_works(): run_scan_click(['--reindex-licenses']) +@pytest.mark.scanslow +def test_detection_with_single_external_license_directory(): + test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) + example1_dir = test_env.get_test_loc('example_external_licenses/example1') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '-dir', example1_dir, + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/external_licenses/scan.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + + +@pytest.mark.scanslow +def test_detection_with_multiple_external_license_directories(): + test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) + example1_dir = test_env.get_test_loc('example_external_licenses/example1') + example2_dir = test_env.get_test_loc('example_external_licenses/example2') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '-dir', example1_dir, + '-dir', example2_dir, + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/external_licenses/scan_multiple.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + + @pytest.mark.scanslow def test_scan_license_with_url_template(): test_dir = test_env.get_test_loc('plugin_license/license_url', copy=True) From 4989205632d9a994d06d167d321f54ae7679e585 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Thu, 23 Jun 2022 09:48:05 -0400 Subject: [PATCH 02/43] Add documentation for new ``--dir`` CLI option Signed-off-by: Kevin Ji --- docs/source/cli-reference/basic-options.rst | 55 +++++++++++++++++++++ docs/source/rst_snippets/basic_options.rst | 7 +++ 2 files changed, 62 insertions(+) diff --git a/docs/source/cli-reference/basic-options.rst b/docs/source/cli-reference/basic-options.rst index 70fbec2c1c3..fa51d944393 100644 --- a/docs/source/cli-reference/basic-options.rst +++ b/docs/source/cli-reference/basic-options.rst @@ -9,6 +9,61 @@ ---- +``-dir, --additional_directories`` Options +------------------------------------------ + + .. admonition:: Dependency + + The option ``-dir, --additional_directories`` is a sub-option of and requires the option + ``--license``. + + The ``-dir, --additional_directories`` option allows the user to include additional directories + of licenses to use in license detection. + + This command only needs to be run once for each set of additional directories; in all subsequent + runs of Scancode with the same directories all the licenses in the directories will be cached. + + The directory structure should look something like this:: + + licenses/ + ├── privateLicense1/ + │ ├── license/ + │ │ ├── privateLicense1.LICENSE + │ │ └── privateLicense1.yml + │ └── rule/ + │ ├── privateLicense1.RULE + │ └── privateLicense1.yml + └── privateLicense2/ + ├── license/ + │ ├── privateLicense2.LICENSE + │ └── privateLicense2.yml + └── rule/ + ├── privateLicense2.RULE + └── privateLicense2.yml + + A scan example using the ``-dir, --additional_directories PATH`` option with a single directory:: + + scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/license1 + + You can also include multiple directories like so:: + + scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/external1 -dir /home/user/external_licenses/external2 + + If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, + you can omit the ``-dir`` option in subsequent scans and they will still be included. :: + + scancode -clpieu --json-pp output.json samples + + However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` + and ``home/user/external_licenses/external3``, you would need to rerun the scan with those directories as parameters. :: + + scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/external1 -dir /home/user/external_licenses/external3 + + + .. + +---- + ``--generated`` Options ----------------------- diff --git a/docs/source/rst_snippets/basic_options.rst b/docs/source/rst_snippets/basic_options.rst index 1b7b8ae885c..e954e34d8a9 100644 --- a/docs/source/rst_snippets/basic_options.rst +++ b/docs/source/rst_snippets/basic_options.rst @@ -55,6 +55,13 @@ documenting a program's options. For example: .. include:: /rst_snippets/note_snippets/basic_clpieu.rst +-dir, --additional_directories PATH + + Include paths to directories containing additional licenses and rules to use + in license detection. This can be used multiple times for multiple directories. + + Sub-Option of - ``--license`` + --generated Classify automatically generated code files with a flag. --max-email INT Report only up to INT emails found in a From 0cbfff1e7ea2bb283469f0fadf44a2a7627f7079 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 25 Jun 2022 15:44:23 -0400 Subject: [PATCH 03/43] Enable using installed licenses in scans #2994 This allows users to use licenses that they install via wheels in license scans. It also adds a sample package containing an external license that can be used to manually verify expected behavior. Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 9 + src/licensedcode/models.py | 21 ++ .../licenses_to_install1/.gitignore | 2 + .../licenses_to_install1/MANIFEST.in | 7 + .../licenses_to_install1/gpl-1.0.LICENSE | 254 ++++++++++++++++++ .../licenses_to_install1/setup.cfg | 6 + .../licenses_to_install1/setup.py | 46 ++++ .../src/licenses_to_install1/__init__.py | 37 +++ .../licenses/exampleInstalled1.LICENSE | 1 + .../licenses/exampleInstalled1.yml | 5 + .../installed_licenses/scan.expected.json | 81 ++++++ .../installed_licenses/scan/license.txt | 1 + 12 files changed, 470 insertions(+) create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml create mode 100644 tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json create mode 100644 tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index feabad893b3..2fb0cece52a 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -362,7 +362,16 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie """ Load or build and cache a LicenseCache. Return None. """ + from licensedcode.models import get_paths_to_installed_licenses_and_rules global _LICENSE_CACHE + + # include installed licenses + if not additional_directories: + additional_directories = get_paths_to_installed_licenses_and_rules() + else: + # additional_directories is originally a tuple + additional_directories = list(additional_directories) + get_paths_to_installed_licenses_and_rules() + if need_cache_rebuild(additional_directories): force = True if force or not _LICENSE_CACHE: diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index a4efef20815..4c56f043da5 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -101,6 +101,8 @@ def logger_debug(*args): CATEGORIES = FOSS_CATEGORIES | OTHER_CATEGORIES +# prefix for name of all externally installed license plugins +EXTERNAL_LICENSE_PLUGIN_PREFIX = 'licenses' @attr.s(slots=True) class License: @@ -792,6 +794,25 @@ def get_rule_dirs( return [f"{str(Path(path).absolute())}/rules" for path in additional_dirs] +def get_paths_to_installed_licenses_and_rules(): + """ + Returns a list of paths to externally packaged licenses or rules that the user has + installed. Gets a list of all of these licenses (installed as plugins) and + then gets the plugins containing licenses by checking that their names start + with a common prefix. + """ + from importlib_metadata import entry_points + from plugincode.location_provider import get_location + installed_plugins = entry_points(group='scancode_location_provider') + paths = [] + for plugin in installed_plugins: + if plugin.name.startswith(EXTERNAL_LICENSE_PLUGIN_PREFIX): + # get path to directory of licenses and/or rules + location_key = plugin.name + paths.append(get_location(location_key)) + return paths + + def load_licenses_from_multiple_dirs( license_directories, with_deprecated=False, diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore new file mode 100644 index 00000000000..95832e8eb97 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore @@ -0,0 +1,2 @@ +/build/ +/dist/ \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in new file mode 100644 index 00000000000..b566ab1f35d --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in @@ -0,0 +1,7 @@ +graft src +graft thirdparty + +include setup.* +include *.rst +include *LICENSE* +include *NOTICE* \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE new file mode 100644 index 00000000000..c510b2875fe --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE @@ -0,0 +1,254 @@ +GNU GENERAL PUBLIC LICENSE + Version 1, February 1989 + + Copyright (C) 1989 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The license agreements of most software companies try to keep users +at the mercy of those companies. By contrast, our General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. The +General Public License applies to the Free Software Foundation's +software and to any other program whose authors commit to using it. +You can use it for your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Specifically, the General Public License is designed to make +sure that you have the freedom to give away or sell copies of free +software, that you receive source code or can get it if you want it, +that you can change the software or use pieces of it in new free +programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of a such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must tell them their rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + The precise terms and conditions for copying, distribution and +modification follow. + + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any program or other work which +contains a notice placed by the copyright holder saying it may be +distributed under the terms of this General Public License. The +"Program", below, refers to any such program or work, and a "work based +on the Program" means either the Program or any work containing the +Program or a portion of it, either verbatim or with modifications. Each +licensee is addressed as "you". + + 1. You may copy and distribute verbatim copies of the Program's source +code as you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this +General Public License and to the absence of any warranty; and give any +other recipients of the Program a copy of this General Public License +along with the Program. You may charge a fee for the physical act of +transferring a copy. + + 2. You may modify your copy or copies of the Program or any portion of +it, and copy and distribute such modifications under the terms of Paragraph +1 above, provided that you also do the following: + + a) cause the modified files to carry prominent notices stating that + you changed the files and the date of any change; and + + b) cause the whole of any work that you distribute or publish, that + in whole or in part contains the Program or any part thereof, either + with or without modifications, to be licensed at no charge to all + third parties under the terms of this General Public License (except + that you may choose to grant warranty protection to some or all + third parties, at your option). + + c) If the modified program normally reads commands interactively when + run, you must cause it, when started running for such interactive use + in the simplest and most usual way, to print or display an + announcement including an appropriate copyright notice and a notice + that there is no warranty (or else, saying that you provide a + warranty) and that users may redistribute the program under these + conditions, and telling the user how to view a copy of this General + Public License. + + d) You may charge a fee for the physical act of transferring a + copy, and you may at your option offer warranty protection in + exchange for a fee. + +Mere aggregation of another independent work with the Program (or its +derivative) on a volume of a storage or distribution medium does not bring +the other work under the scope of these terms. + + + 3. You may copy and distribute the Program (or a portion or derivative of +it, under Paragraph 2) in object code or executable form under the terms of +Paragraphs 1 and 2 above provided that you also do one of the following: + + a) accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of + Paragraphs 1 and 2 above; or, + + b) accompany it with a written offer, valid for at least three + years, to give any third party free (except for a nominal charge + for the cost of distribution) a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of + Paragraphs 1 and 2 above; or, + + c) accompany it with the information you received as to where the + corresponding source code may be obtained. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form alone.) + +Source code for a work means the preferred form of the work for making +modifications to it. For an executable file, complete source code means +all the source code for all modules it contains; but, as a special +exception, it need not include source code for modules which are standard +libraries that accompany the operating system on which the executable +file runs, or for standard header files or definitions files that +accompany that operating system. + + 4. You may not copy, modify, sublicense, distribute or transfer the +Program except as expressly provided under this General Public License. +Any attempt otherwise to copy, modify, sublicense, distribute or transfer +the Program is void, and will automatically terminate your rights to use +the Program under this License. However, parties who have received +copies, or rights to use copies, from you under this General Public +License will not have their licenses terminated so long as such parties +remain in full compliance. + + 5. By copying, distributing or modifying the Program (or any work based +on the Program) you indicate your acceptance of this license to do so, +and all its terms and conditions. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the original +licensor to copy, distribute or modify the Program subject to these +terms and conditions. You may not impose any further restrictions on the +recipients' exercise of the rights granted herein. + + + 7. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of the license which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +the license, you may choose any version ever published by the Free Software +Foundation. + + 8. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + Appendix: How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to humanity, the best way to achieve this is to make it +free software which everyone can redistribute and change under these +terms. + + To do so, attach the following notices to the program. It is safest to +attach them to the start of each source file to most effectively convey +the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) 19yy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 1, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) 19xx name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the +appropriate parts of the General Public License. Of course, the +commands you use may be called something other than `show w' and `show +c'; they could even be mouse-clicks or menu items--whatever suits your +program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + program `Gnomovision' (a program to direct compilers to make passes + at assemblers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +That's all there is to it! \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg new file mode 100644 index 00000000000..b463cb6c31e --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg @@ -0,0 +1,6 @@ +[metadata] +license_files = + gpl-1.0.LICENSE + +[aliases] +release = clean --all bdist_wheel --plat-name manylinux1_x86_64 --python-tag py3 \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py new file mode 100644 index 00000000000..fe37237af0d --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from glob import glob +from os.path import basename +from os.path import join +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +desc = '''A ScanCode path provider plugin to provide a set of example external licenses that can be installed.''' + +setup( + name='licenses_to_install1', + version='1.0', + license=( + 'apache-2.0 AND bsd-simplified-darwin AND (bsd-simplified AND public-domain AND ' + 'bsd-new AND isc AND (bsd-new OR gpl-1.0-plus) AND bsd-original)' + ), + description=desc, + long_description=desc, + author='nexB', + author_email='info@aboutcode.org', + url='https://github.com/nexB/scancode-plugins', + packages=find_packages('src'), + package_dir={'': 'src'}, + include_package_data=True, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Topic :: Utilities', + ], + keywords=[ + 'open source', 'scancode_licenses', + ], + entry_points={ + 'scancode_location_provider': [ + 'licenses_to_install1 = licenses_to_install1:LicensesToInstall1Paths', + ], + }, +) \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py new file mode 100644 index 00000000000..1fdba1cfb9e --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py @@ -0,0 +1,37 @@ +# +# Copyright (c) nexB Inc. and others. +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, this list +# of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, this +# list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from os.path import abspath +from os.path import dirname + +from plugincode.location_provider import LocationProviderPlugin + + +class LicensesToInstall1Paths(LocationProviderPlugin): + def get_locations(self): + curr_dir = dirname(abspath(__file__)) + locations = { + 'licenses_to_install1': curr_dir, + } + return locations diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE new file mode 100644 index 00000000000..f0455cbed1b --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE @@ -0,0 +1 @@ +This is a test license that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml new file mode 100644 index 00000000000..e72e78fdd4b --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml @@ -0,0 +1,5 @@ +key: exampleInstalled1 +short_name: Example Installed License 1 +name: Example Installed License 1 +category: Permissive +owner: NexB \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json new file mode 100644 index 00000000000..6226a744fa7 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -0,0 +1,81 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "-dir": "", + "--json": "", + "--license": true, + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "output_format_version": "2.0.0", + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "linux", + "cpu_architecture": "64", + "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", + "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", + "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + }, + "spdx_license_list_version": "3.16", + "files_count": 2 + } + } + ], + "files": [ + { + "path": "license.txt", + "type": "file", + "licenses": [ + { + "key": "example1", + "score": 100.0, + "name": "Example External License 1", + "short_name": "Example External License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "spdx_license_key": "", + "spdx_url": "", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example1.LICENSE", + "license_expression": "example1", + "licenses": [ + "example1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 9, + "matched_length": 9, + "match_coverage": 100.0, + "rule_relevance": 100 + } + } + ], + "license_expressions": [ + "example1" + ], + "percentage_of_license_text": 10.98, + "scan_errors": [] + } + ] +} diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt b/tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt new file mode 100644 index 00000000000..f0455cbed1b --- /dev/null +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt @@ -0,0 +1 @@ +This is a test license that must be installed into ScanCode Toolkit. From 3376b9cbf81f6abc67352681d0c5bb0d06b7162a Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 10 Jul 2022 10:24:06 -0400 Subject: [PATCH 04/43] Add CI job to test detecting installed license This adds a new CI job in azure-pipelines.yml that installs an external license and tests that we can detect it. Signed-off-by: Kevin Ji --- azure-pipelines.yml | 6 +++ etc/ci/azure-installed-external-license.yml | 38 ++++++++++++++++++ .../installed_licenses/scan.expected.json | 32 +++++++-------- .../test_installed_license_detection.py | 40 +++++++++++++++++++ 4 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 etc/ci/azure-installed-external-license.yml create mode 100644 tests/licensedcode/test_installed_license_detection.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a2962b9a720..fab2a7c1665 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,6 +33,7 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven2.py \ --ignore=tests/licensedcode/test_detection_datadriven3.py \ --ignore=tests/licensedcode/test_detection_datadriven4.py \ + --ignore=tests/licensedcode/test_installed_license_detection.py \ tests/licensedcode license_datadriven1_2: | @@ -168,6 +169,11 @@ jobs: test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + - template: etc/ci/azure-installed-external-license.yml + parameters: + job_name: installed_external_license + image_name: ubuntu-20.04 + python_versions: ['3.8'] ################################################################################ # Test using many version of Click to work around any regressions in their API diff --git a/etc/ci/azure-installed-external-license.yml b/etc/ci/azure-installed-external-license.yml new file mode 100644 index 00000000000..4bd7c257f4e --- /dev/null +++ b/etc/ci/azure-installed-external-license.yml @@ -0,0 +1,38 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + python_architecture: x64 + install_packages: | + venv/bin/python3 -m pip install --upgrade pip setuptools wheel + venv/bin/python3 -m pip install tests/licensedcode/data/example_external_licenses/licenses_to_install1 + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' + + - script: ${{ parameters.install_packages }} + displayName: '${{ pyver }} - Install license' + + - script: venv/bin/pytest -vvs --test-suite=all tests/licensedcode/test_installed_license_detection.py + displayName: '${{ pyver }} - Run tests' \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index 6226a744fa7..590f71f9854 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -22,8 +22,8 @@ "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" }, - "spdx_license_list_version": "3.16", - "files_count": 2 + "spdx_license_list_version": "3.17", + "files_count": 1 } } ], @@ -33,28 +33,28 @@ "type": "file", "licenses": [ { - "key": "example1", + "key": "exampleInstalled1", "score": 100.0, - "name": "Example External License 1", - "short_name": "Example External License 1", + "name": "Example Installed License 1", + "short_name": "Example Installed License 1", "category": "Permissive", "is_exception": false, "is_unknown": false, "owner": "NexB", "homepage_url": "", "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "reference_url": "https://scancode-licensedb.aboutcode.org/exampleInstalled1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/exampleInstalled1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/exampleInstalled1.yml", "spdx_license_key": "", "spdx_url": "", "start_line": 1, "end_line": 1, "matched_rule": { - "identifier": "example1.LICENSE", - "license_expression": "example1", + "identifier": "exampleInstalled1.LICENSE", + "license_expression": "exampleInstalled1", "licenses": [ - "example1" + "exampleInstalled1" ], "referenced_filenames": [], "is_license_text": true, @@ -63,18 +63,18 @@ "is_license_tag": false, "is_license_intro": false, "has_unknown": false, - "matcher": "2-aho", - "rule_length": 9, - "matched_length": 9, + "matcher": "1-hash", + "rule_length": 11, + "matched_length": 11, "match_coverage": 100.0, "rule_relevance": 100 } } ], "license_expressions": [ - "example1" + "exampleInstalled1" ], - "percentage_of_license_text": 10.98, + "percentage_of_license_text": 100.0, "scan_errors": [] } ] diff --git a/tests/licensedcode/test_installed_license_detection.py b/tests/licensedcode/test_installed_license_detection.py new file mode 100644 index 00000000000..1878ea3a0d9 --- /dev/null +++ b/tests/licensedcode/test_installed_license_detection.py @@ -0,0 +1,40 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +import pytest +from commoncode.testcase import FileDrivenTesting + +from scancode.cli_test_utils import check_json_scan +from scancode.cli_test_utils import run_scan_click +from scancode_config import REGEN_TEST_FIXTURES + +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + +""" +These tests spawn new process as if launched from the command line. +""" + + +@pytest.mark.scanslow +def test_detection_with_single_installed_external_license(): + test_dir = test_env.get_test_loc('plugin_license/installed_licenses/scan', copy=True) + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/installed_licenses/scan.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) From 681436da62d77b3b56d7e7a23629bf03fc38a75a Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Fri, 15 Jul 2022 14:56:55 -0400 Subject: [PATCH 05/43] Add documentation for installed license plugins This adds documentation for how to set up directories of installed licenses and how to install them for license detection. Signed-off-by: Kevin Ji --- docs/source/how-to-guides/index.rst | 1 + .../install_new_license_plugin.rst | 81 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docs/source/how-to-guides/install_new_license_plugin.rst diff --git a/docs/source/how-to-guides/index.rst b/docs/source/how-to-guides/index.rst index 957445a3219..ef2604190c9 100644 --- a/docs/source/how-to-guides/index.rst +++ b/docs/source/how-to-guides/index.rst @@ -8,3 +8,4 @@ add_new_license add_new_license_detection_rule + install_new_license_plugin diff --git a/docs/source/how-to-guides/install_new_license_plugin.rst b/docs/source/how-to-guides/install_new_license_plugin.rst new file mode 100644 index 00000000000..31c1e86cd70 --- /dev/null +++ b/docs/source/how-to-guides/install_new_license_plugin.rst @@ -0,0 +1,81 @@ +.. _install_new_license_plugin: + +How to Install External License Plugins to Use in License Detection +=================================================================== + +Users can install external licenses and rules in the form of plugins. These +licenses and rules are then used in license detection. + +How to create a plugin containing external licenses and/or rules +---------------------------------------------------------------- + +To create a plugin with external licenses or rules, we must create a Python package +containing the license and/or rule files. Python packages can have many different +file structures. You can find an example package in +``tests/licensedcode/data/example_external_licenses/licenses_to_install1``. + +This is the basic structure of the example plugin:: + + licenses_to_install1/ + ├── src/ + │ └── licenses_to_install1/ + │ ├── licenses/ + │ │ ├── example_installed_1.LICENSE + │ │ └── example_installed_1.yaml + | ├── rules/ + │ │ ├── example_installed_1.RULE + │ │ └── example_installed_1.yaml + │ └── __init__.py + ├── gpl-1.0.LICENSE + ├── MANIFEST.in + ├── setup.cfg + └── setup.py + +Key points to note +------------------ + +Entry points definition in ``setup.py`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +First, in ``setup.py``, you must provide an entry point called ``scancode_location_provider``. +This allows ScanCode Toolkit to discover the plugin and use it in license detection. +Here is the definition of ``entry_points`` in ``setup.py``:: + + entry_points={ + 'scancode_location_provider': [ + 'licenses_to_install1 = licenses_to_install1:LicensesToInstall1Paths', + ], + }, + +The ``scancode_location_provider`` entry point maps to a list with information about the plugin. +The variable ``licenses_to_install1`` is the name of the entry point. All entry point names +**must** start with the prefix ``licenses``, or else ScanCode Toolkit will not use them in +license detection. + +Directory structure +^^^^^^^^^^^^^^^^^^^ + +``licenses_to_install1`` is set to ``licenses_to_install1:LicensesToInstall1Paths``. +Note that in ``src``, we have another directory called ``licenses_to_install1`` and in +``licenses_to_install1/__init__.py``, we define the class ``LicensesToInstall1Paths``. +These two values make up the entry point definition. + +``LicensesToInstall1Paths`` is a subclass of ``LocationProviderPlugin`` and +implements the method ``get_locations()``. The class you define in ``__init__.py`` +must also subclass ``LocationProviderPlugin`` and implement this method. + +Finally, the same directory containing the class definition must also contain the +licenses and/or rules. Licenses must be contained in a directory called ``licenses`` and rules +must be contained in a directory called ``rules``. + +See :ref:`add_new_license_for_det` and :ref:`add_new_license_det_rule` to understand +the structure of license and rule files, respectively. + +After creating this plugin, you can upload it to PyPI so that others can use it, or you can +leave it as a local directory. + +Installing and using the plugin +------------------------------- +To use the plugin in license detection, all you need to do is install it using ``pip``. +Once it is installed, the contained licenses and rules will automatically be used in +license detection assuming the plugin follows the correct directory structure conventions. From e741ffd93722342b4585902b9c3d1fe320866b3f Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 16 Jul 2022 13:51:29 -0400 Subject: [PATCH 06/43] Enable installed rules to be used in detection This adds files in the example installed license that can be used in license detection. It also adds a test to be run as part of a CI job. Signed-off-by: Kevin Ji --- ...d1.LICENSE => example-installed-1.LICENSE} | 0 ...Installed1.yml => example-installed-1.yml} | 2 +- .../rules/example-installed-1.RULE | 1 + .../rules/example-installed-1.yml | 2 + .../installed_licenses/scan.expected.json | 20 ++--- .../installed_rules/scan.expected.json | 81 +++++++++++++++++++ .../installed_rules/scan/license.txt | 1 + .../test_installed_license_detection.py | 16 ++++ 8 files changed, 112 insertions(+), 11 deletions(-) rename tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/{exampleInstalled1.LICENSE => example-installed-1.LICENSE} (100%) rename tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/{exampleInstalled1.yml => example-installed-1.yml} (80%) create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml create mode 100644 tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json create mode 100644 tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.LICENSE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.LICENSE rename to tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.LICENSE diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml similarity index 80% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml rename to tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml index e72e78fdd4b..680e629f78d 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/exampleInstalled1.yml +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml @@ -1,4 +1,4 @@ -key: exampleInstalled1 +key: example-installed-1 short_name: Example Installed License 1 name: Example Installed License 1 category: Permissive diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE new file mode 100644 index 00000000000..bb701de0a88 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE @@ -0,0 +1 @@ +This is a test rule that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml new file mode 100644 index 00000000000..77dde8a776f --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml @@ -0,0 +1,2 @@ +license_expression: example-installed-1 +is_license_text: yes diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index 590f71f9854..a55d43347bf 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -33,7 +33,7 @@ "type": "file", "licenses": [ { - "key": "exampleInstalled1", + "key": "example-installed-1", "score": 100.0, "name": "Example Installed License 1", "short_name": "Example Installed License 1", @@ -43,18 +43,18 @@ "owner": "NexB", "homepage_url": "", "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/exampleInstalled1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/exampleInstalled1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/exampleInstalled1.yml", - "spdx_license_key": "", - "spdx_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml", + "spdx_license_key": "scancode-example-installed1", + "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, "end_line": 1, "matched_rule": { - "identifier": "exampleInstalled1.LICENSE", - "license_expression": "exampleInstalled1", + "identifier": "example-installed-1.LICENSE", + "license_expression": "example-installed-1", "licenses": [ - "exampleInstalled1" + "example-installed-1" ], "referenced_filenames": [], "is_license_text": true, @@ -72,7 +72,7 @@ } ], "license_expressions": [ - "exampleInstalled1" + "example-installed-1" ], "percentage_of_license_text": 100.0, "scan_errors": [] diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json new file mode 100644 index 00000000000..c7bbbcbe38f --- /dev/null +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json @@ -0,0 +1,81 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "-dir": "", + "--json": "", + "--license": true, + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "output_format_version": "2.0.0", + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "linux", + "cpu_architecture": "64", + "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", + "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", + "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + }, + "spdx_license_list_version": "3.17", + "files_count": 1 + } + } + ], + "files": [ + { + "path": "license.txt", + "type": "file", + "licenses": [ + { + "key": "example-installed-1", + "score": 61.0, + "name": "Example Installed License 1", + "short_name": "Example Installed License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml", + "spdx_license_key": "", + "spdx_url": "", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example-installed-1.RULE", + "license_expression": "example-installed-1", + "licenses": [ + "example-installed-1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "1-hash", + "rule_length": 11, + "matched_length": 11, + "match_coverage": 100.0, + "rule_relevance": 61 + } + } + ], + "license_expressions": [ + "example-installed-1" + ], + "percentage_of_license_text": 100.0, + "scan_errors": [] + } + ] +} diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt b/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt new file mode 100644 index 00000000000..bb701de0a88 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt @@ -0,0 +1 @@ +This is a test rule that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/test_installed_license_detection.py b/tests/licensedcode/test_installed_license_detection.py index 1878ea3a0d9..10c13e14f23 100644 --- a/tests/licensedcode/test_installed_license_detection.py +++ b/tests/licensedcode/test_installed_license_detection.py @@ -38,3 +38,19 @@ def test_detection_with_single_installed_external_license(): run_scan_click(args) test_loc = test_env.get_test_loc('plugin_license/installed_licenses/scan.expected.json') check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + + +@pytest.mark.scanslow +def test_detection_with_single_installed_external_rule(): + test_dir = test_env.get_test_loc('plugin_license/installed_rules/scan', copy=True) + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/installed_rules/scan.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) From afae69285341151d85693021f2af119ff7e3fa85 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Mon, 18 Jul 2022 11:26:27 -0400 Subject: [PATCH 07/43] Move `licensedcode_test_utils` into main wheel This moves `licensedcode_test_utils` from tests into src so that users can write their own tests for installed licenses. It also adds documentation for how to write and run these tests. Signed-off-by: Kevin Ji --- .../install_new_license_plugin.rst | 66 +++++++++++++++++-- .../licensedcode_test_utils.py | 0 .../licenses_to_install1/setup.py | 6 +- .../tests/data/example-installed-1.txt | 8 +++ .../tests/data/example-installed-1.txt.yml | 2 + .../tests/test_detection_datadriven.py | 34 ++++++++++ 6 files changed, 111 insertions(+), 5 deletions(-) rename {tests/licensedcode => src}/licensedcode_test_utils.py (100%) create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py diff --git a/docs/source/how-to-guides/install_new_license_plugin.rst b/docs/source/how-to-guides/install_new_license_plugin.rst index 31c1e86cd70..cd3d1756f3f 100644 --- a/docs/source/how-to-guides/install_new_license_plugin.rst +++ b/docs/source/how-to-guides/install_new_license_plugin.rst @@ -20,11 +20,11 @@ This is the basic structure of the example plugin:: ├── src/ │ └── licenses_to_install1/ │ ├── licenses/ - │ │ ├── example_installed_1.LICENSE - │ │ └── example_installed_1.yaml + │ │ ├── example-installed-1.LICENSE + │ │ └── example-installed-1.yaml | ├── rules/ - │ │ ├── example_installed_1.RULE - │ │ └── example_installed_1.yaml + │ │ ├── example-installed-1.RULE + │ │ └── example-installed-1.yaml │ └── __init__.py ├── gpl-1.0.LICENSE ├── MANIFEST.in @@ -79,3 +79,61 @@ Installing and using the plugin To use the plugin in license detection, all you need to do is install it using ``pip``. Once it is installed, the contained licenses and rules will automatically be used in license detection assuming the plugin follows the correct directory structure conventions. + +Writing tests for new installed licenses +---------------------------------------- + +Look at ``tests/licensedcode/data/example_external_licenses/licenses_to_install1`` to see +an example of a plugin with tests. The tests are contained in the ``tests`` directory:: + + licenses_to_install1/ + ├── src/ + │ └── licenses_to_install1/ + │ ├── licenses/ + │ │ ├── example-installed-1.LICENSE + │ │ └── example-installed-1.yaml + │ ├── rules/ + │ │ ├── example-installed-1.RULE + │ │ └── example-installed-1.yaml + │ └── __init__.py/ + ├── tests/ + │ ├── data/ + │ │ ├── example-installed-1.txt + │ │ └── example-installed-1.txt.yml + │ └── test_detection_datadriven.py + ├── gpl-1.0.LICENSE + ├── MANIFEST.in + ├── setup.cfg + └── setup.py + +To write your own tests, first make sure ``setup.py`` includes ``scancode-toolkit`` +as a dependency:: + + ... + install_requires=[ + 'scancode-toolkit', + ], + ... + +Then you can define a test class and call the ``build_tests`` method defined in +``licensedcode_test_utils``, passing in the test directory and the test class as parameters:: + + TEST_DIR = abspath(join(dirname(__file__), 'data')) + + + class TestLicenseDataDriven1(unittest.TestCase): + pass + + + licensedcode_test_utils.build_tests( + TEST_DIR, + clazz=TestLicenseDataDriven1, regen=scancode_config.REGEN_TEST_FIXTURES) + +The ``tests/data`` directory contains a pair of files for each license: +a license text file and a YAML file specifying the expected license expressions from the test. + +Finally, to run the test, do the following: + +1. Create a virtual environment to install the package into. +2. Install the package using ``pip``, e.g. ``pip install ./licenses_to_install1``. +3. Run the tests, e.g. ``py.test tests/test_detection_datadriven.py``. diff --git a/tests/licensedcode/licensedcode_test_utils.py b/src/licensedcode_test_utils.py similarity index 100% rename from tests/licensedcode/licensedcode_test_utils.py rename to src/licensedcode_test_utils.py diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py index fe37237af0d..d947bf9f3b7 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py @@ -3,7 +3,6 @@ from glob import glob from os.path import basename -from os.path import join from os.path import splitext from setuptools import find_packages @@ -38,6 +37,11 @@ keywords=[ 'open source', 'scancode_licenses', ], + install_requires=[ + 'scancode-toolkit', + 'wheel', + 'pytest', + ], entry_points={ 'scancode_location_provider': [ 'licenses_to_install1 = licenses_to_install1:LicensesToInstall1Paths', diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt new file mode 100644 index 00000000000..932b6e6357f --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt @@ -0,0 +1,8 @@ +This is a test license that must be installed into ScanCode Toolkit. +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor +incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud +exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure +dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit +anim id est laborum. + diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml new file mode 100644 index 00000000000..52d347c614c --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml @@ -0,0 +1,2 @@ +license_expressions: + - example-installed-1 diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py new file mode 100644 index 00000000000..c375c0417f2 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py @@ -0,0 +1,34 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from os.path import abspath +from os.path import join +from os.path import dirname +import unittest + +from licensedcode_test_utils import build_tests # NOQA +from scancode_config import REGEN_TEST_FIXTURES + + + +""" +Data-driven tests using expectations stored in YAML files. +Test functions are attached to test classes at module import time +""" + +TEST_DIR = abspath(join(dirname(__file__), 'data')) + + +class TestLicenseDataDriven(unittest.TestCase): + pass + + +build_tests( + TEST_DIR, + clazz=TestLicenseDataDriven, regen=REGEN_TEST_FIXTURES) From f576dfa4ec395e9f1e892613a3d5e85f3ef486bf Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Wed, 20 Jul 2022 22:42:40 -0400 Subject: [PATCH 08/43] Add Windows and MacOS images to Azure pipelines This adds Windows and MacOS images as part of the CI job to test the installed external license detection. Signed-off-by: Kevin Ji --- azure-pipelines.yml | 16 +++++++- ...zure-posix-installed-external-license.yml} | 0 .../azure-win-installed-external-license.yml | 38 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) rename etc/ci/{azure-installed-external-license.yml => azure-posix-installed-external-license.yml} (100%) create mode 100644 etc/ci/azure-win-installed-external-license.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fab2a7c1665..6cd7936fef5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -169,12 +169,24 @@ jobs: test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - template: etc/ci/azure-installed-external-license.yml + - template: etc/ci/azure-posix-installed-external-license.yml parameters: - job_name: installed_external_license + job_name: installed_external_license_ubuntu image_name: ubuntu-20.04 python_versions: ['3.8'] + - template: etc/ci/azure-win-installed-external-license.yml + parameters: + job_name: installed_external_license_windows + image_name: windows-2022 + python_versions: ['3.8'] + + - template: etc/ci/azure-posix-installed-external-license.yml + parameters: + job_name: installed_external_license_macos11 + image_name: macos-11 + python_versions: ['3.8'] + ################################################################################ # Test using many version of Click to work around any regressions in their API ################################################################################ diff --git a/etc/ci/azure-installed-external-license.yml b/etc/ci/azure-posix-installed-external-license.yml similarity index 100% rename from etc/ci/azure-installed-external-license.yml rename to etc/ci/azure-posix-installed-external-license.yml diff --git a/etc/ci/azure-win-installed-external-license.yml b/etc/ci/azure-win-installed-external-license.yml new file mode 100644 index 00000000000..c8b99bf2371 --- /dev/null +++ b/etc/ci/azure-win-installed-external-license.yml @@ -0,0 +1,38 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + python_architecture: x64 + install_packages: | + venv\Scripts\python.exe -m pip install --upgrade pip setuptools wheel + venv\Scripts\python.exe -m pip install tests\licensedcode\data\example_external_licenses\licenses_to_install1 + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' + + - script: ${{ parameters.install_packages }} + displayName: '${{ pyver }} - Install license' + + - script: venv\Scripts\pytest -vvs --test-suite=all tests\licensedcode\test_installed_license_detection.py + displayName: '${{ pyver }} - Run tests' From a68c4cc0ac8d5a132fae714cc22f7b3cc0e09c22 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Thu, 21 Jul 2022 20:04:15 -0400 Subject: [PATCH 09/43] Add rule and license validation when index is made This performs the checks `check_ignorable_clues()` in test_detection_validate.py and `test_validate_license_library_data()` in test_models.py when the index is created. This will allow us to validate additional rules and licenses that are added into the index. Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 4 + src/licensedcode/models.py | 73 +++++++++++++++++++ .../licenses/apache-2.0.LICENSE | 5 ++ .../validate_licenses/licenses/apache-2.0.yml | 11 +++ .../licenses/bsd-ack-carrot2.LICENSE | 5 ++ .../licenses/bsd-ack-carrot2.yml | 1 + tests/licensedcode/test_plugin_license.py | 20 +++++ 7 files changed, 119 insertions(+) create mode 100644 tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE create mode 100644 tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml create mode 100644 tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE create mode 100644 tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 2fb0cece52a..1b194085e4f 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -102,6 +102,7 @@ def load_or_build( from licensedcode.models import load_licenses from licensedcode.models import load_licenses_from_multiple_dirs from licensedcode.models import get_license_dirs + from licensedcode.models import validate_additional_license_data from scancode import lockfile licenses_data_dir = licenses_data_dir or ldd @@ -118,6 +119,7 @@ def load_or_build( if additional_directories: additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) + validate_additional_license_data(additional_license_dirs) combined_directories = [licenses_data_dir] + additional_license_dirs licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories) else: @@ -187,6 +189,7 @@ def build_index( from licensedcode.models import rules_data_dir as rdd from licensedcode.models import load_licenses from licensedcode.models import load_licenses_from_multiple_dirs + from licensedcode.models import validate_ignorable_clues from licensedcode.legalese import common_license_words licenses_data_dir = licenses_data_dir or ldd @@ -205,6 +208,7 @@ def build_index( if additional_directories: # if we have additional directories, extract the rules from them additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) + validate_ignorable_clues(additional_rule_dirs) # then combine the rules in these additional directories with the rules in the original rules directory combined_rule_directories = [rules_data_dir] + additional_rule_dirs rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index 4c56f043da5..55908468f02 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -854,6 +854,79 @@ def get_rules_from_multiple_dirs( return get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) +class InvalidLicense(Exception): + pass + + +def validate_additional_license_data(additional_directories): + """ + Takes in directories of additional licenses and determines whether they are valid. + If there are any invalid licenses, raises an exception. + """ + licenses = load_licenses_from_multiple_dirs(additional_directories) + errors, _, _ = License.validate( + licenses, + verbose=False, + ) + if errors: + message = ['Errors while validating licenses:'] + for key, msgs in errors.items(): + message.append('') + message.append(f'License: {key}') + for msg in msgs: + message.append(f' {msg!r}') + raise InvalidLicense('\n'.join(message)) + + +def _ignorable_clue_error(rule): + """ + Helper method to validate a single rule's ignorable clues. + Returns a pair of the result and expected ignorable clues if + there is an error. Otherwise, returns None. + """ + result = get_ignorables(rule.text_file) + expected = get_normalized_ignorables(rule) + if result != expected: + data_file = rule.data_file + if not data_file: + data_file = rule.text_file.replace('.LICENSE', '.yml') + + result['files'] = [ + f'file://{data_file}', + f'file://{rule.text_file}', + ] + return result, expected + + +def validate_ignorable_clues(rule_directories): + """ + Validates that all expected ignorable clues declared in a Rule + are properly detected in the rule text file. + """ + combined_rules = [] + for rules_dir in rule_directories: + r = list(load_rules( + rules_data_dir=rules_dir, + )) + combined_rules.append(r) + # flatten lists of rules into a single iterable + rules = list(chain.from_iterable(combined_rules)) + messages = ['Errors while validating ignorable rules:'] + error_present = False + for rule in rules: + if _ignorable_clue_error(rule): + error_present = True + result, expected = _ignorable_clue_error(rule) + message.append('') + message.append(f'{rule!r}') + message.append('Result:') + message.append(result) + message.append('Expected:') + message.append(expected) + if error_present: + raise InvalidRule('\n'.join(message)) + + class InvalidRule(Exception): pass diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE new file mode 100644 index 00000000000..7348616c691 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE @@ -0,0 +1,5 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml new file mode 100644 index 00000000000..abeb5d2fb3c --- /dev/null +++ b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml @@ -0,0 +1,11 @@ +key: apache-2.0 +short_name: Apache 2.0 +name: Apache License 2.0 +category: Permissive +owner: Apache Software Foundation +homepage_url: http://www.apache.org/licenses/ +spdx_license_key: Apache-2.0 +text_urls: + - http://www.apache.org/licenses/LICENSE-2.0 +osi_url: http://opensource.org/licenses/apache2.0.php +faq_url: http://www.apache.org/foundation/licence-FAQ.html diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE new file mode 100644 index 00000000000..7348616c691 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE @@ -0,0 +1,5 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml new file mode 100644 index 00000000000..2f7cbaf8fab --- /dev/null +++ b/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml @@ -0,0 +1 @@ +key: bsd-ack-carrot2 diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 5eada5c99d9..b5199cf3fdf 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -313,3 +313,23 @@ def test_detection_is_correct_in_legacy_npm_package_json(): expected_file = test_env.get_test_loc('plugin_license/package/package.expected.json') run_scan_click(['-lp', '--json-pp', result_file, test_dir]) check_json_scan(expected_file, result_file, remove_uuid=True, remove_file_date=True, regen=REGEN_TEST_FIXTURES) + + +def test_validate_license_library_returns_errors(): + test_dir = test_env.get_test_loc('plugin_license/license-expression/scan', copy=True) + licenses_dir = test_env.get_test_loc('plugin_license/validate_licenses') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + '-dir', licenses_dir, + test_dir, + ] + # the actual error is an InvalidLicense, but run_scan_click has an assert so it + # raises an AssertionError when the code itself throws an InvalidLicense error. + # This is why we assert the test raises an AssertionError but check that the + # error text is an InvalidLicense. + with pytest.raises(AssertionError, match=r"InvalidLicense"): + run_scan_click(args) From 789cf76b11876553587562b1eb25e7aa248556fd Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 24 Jul 2022 20:45:57 -0400 Subject: [PATCH 10/43] Add SPDX license key to example licenses Signed-off-by: Kevin Ji --- .../example_external_licenses/example1/licenses/example1.yml | 3 ++- .../example_external_licenses/example2/licenses/example2.yml | 3 ++- .../src/licenses_to_install1/licenses/example-installed-1.yml | 3 ++- .../data/plugin_license/installed_rules/scan.expected.json | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml index d7d1ea640ec..0977bcdb733 100644 --- a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml +++ b/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml @@ -2,4 +2,5 @@ key: example1 short_name: Example External License 1 name: Example External License 1 category: Permissive -owner: NexB \ No newline at end of file +owner: NexB +spdx_license_key: scancode-example1 diff --git a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml index d255bd8d70e..962caeb2f91 100644 --- a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml +++ b/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml @@ -2,4 +2,5 @@ key: example2 short_name: Example External License 2 name: Example External License 2 category: Permissive -owner: NexB \ No newline at end of file +owner: NexB +spdx_license_key: scancode-example2 diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml index 680e629f78d..df3f186d537 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml @@ -2,4 +2,5 @@ key: example-installed-1 short_name: Example Installed License 1 name: Example Installed License 1 category: Permissive -owner: NexB \ No newline at end of file +owner: NexB +spdx_license_key: scancode-example-installed1 diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json index c7bbbcbe38f..f7615ae1693 100644 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json @@ -46,8 +46,8 @@ "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml", - "spdx_license_key": "", - "spdx_url": "", + "spdx_license_key": "scancode-example-installed1", + "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, "end_line": 1, "matched_rule": { From e7809eec0185122f3812bdc60e24b2e3fd15f807 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Tue, 9 Aug 2022 19:40:11 -0700 Subject: [PATCH 11/43] Refactor CLI option for external licenses This removes ``-dir`` as an option and renames ``--additional_directories`` to ``--external-license-directory``. Signed-off-by: Kevin Ji --- docs/source/cli-reference/basic-options.rst | 18 +++++++++--------- docs/source/rst_snippets/basic_options.rst | 2 +- src/licensedcode/plugin_license.py | 4 ++-- .../external_licenses/scan.expected.json | 2 +- .../scan_multiple.expected.json | 2 +- .../installed_licenses/scan.expected.json | 1 - .../installed_rules/scan.expected.json | 1 - tests/licensedcode/test_plugin_license.py | 8 ++++---- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/source/cli-reference/basic-options.rst b/docs/source/cli-reference/basic-options.rst index fa51d944393..eb0db5686e4 100644 --- a/docs/source/cli-reference/basic-options.rst +++ b/docs/source/cli-reference/basic-options.rst @@ -9,15 +9,15 @@ ---- -``-dir, --additional_directories`` Options ------------------------------------------- +``--external-license-directory`` Options +---------------------------------------- .. admonition:: Dependency - The option ``-dir, --additional_directories`` is a sub-option of and requires the option + The option ``--external-license-directory`` is a sub-option of and requires the option ``--license``. - The ``-dir, --additional_directories`` option allows the user to include additional directories + The ``--external-license-directory`` option allows the user to include additional directories of licenses to use in license detection. This command only needs to be run once for each set of additional directories; in all subsequent @@ -41,23 +41,23 @@ ├── privateLicense2.RULE └── privateLicense2.yml - A scan example using the ``-dir, --additional_directories PATH`` option with a single directory:: + A scan example using the ``--external-license-directory PATH`` option with a single directory:: - scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/license1 + scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/license1 You can also include multiple directories like so:: - scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/external1 -dir /home/user/external_licenses/external2 + scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external2 If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, - you can omit the ``-dir`` option in subsequent scans and they will still be included. :: + you can omit the ``--external-license-directory`` option in subsequent scans and they will still be included. :: scancode -clpieu --json-pp output.json samples However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` and ``home/user/external_licenses/external3``, you would need to rerun the scan with those directories as parameters. :: - scancode -clpieu --json-pp output.json samples -dir /home/user/external_licenses/external1 -dir /home/user/external_licenses/external3 + scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external3 .. diff --git a/docs/source/rst_snippets/basic_options.rst b/docs/source/rst_snippets/basic_options.rst index e954e34d8a9..cbc634effc6 100644 --- a/docs/source/rst_snippets/basic_options.rst +++ b/docs/source/rst_snippets/basic_options.rst @@ -55,7 +55,7 @@ documenting a program's options. For example: .. include:: /rst_snippets/note_snippets/basic_clpieu.rst --dir, --additional_directories PATH +--external-license-directory PATH Include paths to directories containing additional licenses and rules to use in license detection. This can be used multiple times for multiple directories. diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index ff1ba3fc6d8..1586a6bc21a 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -141,7 +141,7 @@ class LicenseScanner(ScanPlugin): ), PluggableCommandLineOption( - ('-dir', '--additional_directories'), + ('--external-license-directory',), required_options=['license'], multiple=True, type=click.Path(exists=True, readable=True, path_type=str), @@ -177,7 +177,7 @@ def setup(self, **kwargs): loaded index. """ from licensedcode.cache import populate_cache - additional_directories = kwargs.get('additional_directories') + additional_directories = kwargs.get('external_license_directory') populate_cache(additional_directories=additional_directories) def get_scanner( diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json index cc9b8a31704..abb56159e26 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -4,7 +4,7 @@ "tool_name": "scancode-toolkit", "options": { "input": "", - "-dir": "", + "--external-license-directory": "", "--json": "", "--license": true, "--strip-root": true diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index e0128e97e5d..d5f19b0e8f8 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -4,7 +4,7 @@ "tool_name": "scancode-toolkit", "options": { "input": "", - "-dir": "", + "--external-license-directory": "", "--json": "", "--license": true, "--strip-root": true diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index a55d43347bf..ad21fd701ca 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -4,7 +4,6 @@ "tool_name": "scancode-toolkit", "options": { "input": "", - "-dir": "", "--json": "", "--license": true, "--strip-root": true diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json index f7615ae1693..07e48d87e10 100644 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json @@ -4,7 +4,6 @@ "tool_name": "scancode-toolkit", "options": { "input": "", - "-dir": "", "--json": "", "--license": true, "--strip-root": true diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index b5199cf3fdf..d5ca0a6c1d2 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -251,7 +251,7 @@ def test_detection_with_single_external_license_directory(): '--license', '--strip-root', '--verbose', - '-dir', example1_dir, + '--external-license-directory', example1_dir, '--json', result_file, test_dir, ] @@ -270,8 +270,8 @@ def test_detection_with_multiple_external_license_directories(): '--license', '--strip-root', '--verbose', - '-dir', example1_dir, - '-dir', example2_dir, + '--external-license-directory', example1_dir, + '--external-license-directory', example2_dir, '--json', result_file, test_dir, ] @@ -324,7 +324,7 @@ def test_validate_license_library_returns_errors(): '--strip-root', '--verbose', '--json', result_file, - '-dir', licenses_dir, + '--external-license-directory', licenses_dir, test_dir, ] # the actual error is an InvalidLicense, but run_scan_click has an assert so it From cdf627f98255eddfcc69f2929bdbb6a649436458 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 27 Aug 2022 15:15:38 -0700 Subject: [PATCH 12/43] revise documentation for --additional-license-directory Signed-off-by: Kevin Ji --- docs/source/cli-reference/basic-options.rst | 51 ------------------- docs/source/cli-reference/core-options.rst | 51 +++++++++++++++++++ docs/source/rst_snippets/basic_options.rst | 7 +-- docs/source/rst_snippets/core_options.rst | 5 ++ .../rst_snippets/note_snippets/core_indep.rst | 3 +- 5 files changed, 59 insertions(+), 58 deletions(-) diff --git a/docs/source/cli-reference/basic-options.rst b/docs/source/cli-reference/basic-options.rst index eb0db5686e4..561e4659723 100644 --- a/docs/source/cli-reference/basic-options.rst +++ b/docs/source/cli-reference/basic-options.rst @@ -9,58 +9,7 @@ ---- -``--external-license-directory`` Options ----------------------------------------- - .. admonition:: Dependency - - The option ``--external-license-directory`` is a sub-option of and requires the option - ``--license``. - - The ``--external-license-directory`` option allows the user to include additional directories - of licenses to use in license detection. - - This command only needs to be run once for each set of additional directories; in all subsequent - runs of Scancode with the same directories all the licenses in the directories will be cached. - - The directory structure should look something like this:: - - licenses/ - ├── privateLicense1/ - │ ├── license/ - │ │ ├── privateLicense1.LICENSE - │ │ └── privateLicense1.yml - │ └── rule/ - │ ├── privateLicense1.RULE - │ └── privateLicense1.yml - └── privateLicense2/ - ├── license/ - │ ├── privateLicense2.LICENSE - │ └── privateLicense2.yml - └── rule/ - ├── privateLicense2.RULE - └── privateLicense2.yml - - A scan example using the ``--external-license-directory PATH`` option with a single directory:: - - scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/license1 - - You can also include multiple directories like so:: - - scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external2 - - If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, - you can omit the ``--external-license-directory`` option in subsequent scans and they will still be included. :: - - scancode -clpieu --json-pp output.json samples - - However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` - and ``home/user/external_licenses/external3``, you would need to rerun the scan with those directories as parameters. :: - - scancode -clpieu --json-pp output.json samples --external-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external3 - - - .. ---- diff --git a/docs/source/cli-reference/core-options.rst b/docs/source/cli-reference/core-options.rst index a9650b97811..f9fdf7e6bbe 100644 --- a/docs/source/cli-reference/core-options.rst +++ b/docs/source/cli-reference/core-options.rst @@ -83,6 +83,57 @@ Comparing Progress Message Options .. [ToDo] Research and Write Better +``--additional-license-directory`` Options +---------------------------------------- + + .. admonition:: Dependency + + The option ``--additional-license-directory`` requires the option ``-reindex--licenses``. + + The ``--additional-license-directory`` option allows the user to include additional directories + of licenses to use in license detection. + + This command only needs to be run once for each set of additional directories; in all subsequent + runs of Scancode with the same directories all the licenses in the directories will be cached. + + The directory structure should look something like this:: + + licenses/ + ├── privateLicense1/ + │ ├── license/ + │ │ ├── privateLicense1.LICENSE + │ │ └── privateLicense1.yml + │ └── rule/ + │ ├── privateLicense1.RULE + │ └── privateLicense1.yml + └── privateLicense2/ + ├── license/ + │ ├── privateLicense2.LICENSE + │ └── privateLicense2.yml + └── rule/ + ├── privateLicense2.RULE + └── privateLicense2.yml + + Here is an example of reindexing the license cache using the ``--additional-license-directory PATH`` option with a single directory:: + + scancode --reindex-licenses --additional-license-directory /home/user/external_licenses/license1 + + You can also include multiple directories like so:: + + scancode --reindex-licenses --additional-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external2 + + If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, + you can simply run scans after reindexing with those directories and they will be included. :: + + scancode -clpieu --json-pp output.json samples + + However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` + and ``home/user/external_licenses/external3``, you would need to reindex the license index with those directories as parameters. :: + + scancode --additional-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external3 + + .. + ---- ``--from-json`` Option diff --git a/docs/source/rst_snippets/basic_options.rst b/docs/source/rst_snippets/basic_options.rst index cbc634effc6..d5394b3fda9 100644 --- a/docs/source/rst_snippets/basic_options.rst +++ b/docs/source/rst_snippets/basic_options.rst @@ -55,12 +55,7 @@ documenting a program's options. For example: .. include:: /rst_snippets/note_snippets/basic_clpieu.rst ---external-license-directory PATH - - Include paths to directories containing additional licenses and rules to use - in license detection. This can be used multiple times for multiple directories. - - Sub-Option of - ``--license`` +` --generated Classify automatically generated code files with a flag. diff --git a/docs/source/rst_snippets/core_options.rst b/docs/source/rst_snippets/core_options.rst index 8845f37316d..3ed523415ab 100644 --- a/docs/source/rst_snippets/core_options.rst +++ b/docs/source/rst_snippets/core_options.rst @@ -14,6 +14,11 @@ All "Core" Scan Options --reindex-licenses Force a check and possible reindexing of the cached license index. +--additional-license-directory PATH + + Include paths to directories containing additional licenses and rules to use + in license detection. This can be used multiple times for multiple directories. + --from-json Load codebase from an existing JSON scan --max-in-memory INTEGER Maximum number of files and directories scan diff --git a/docs/source/rst_snippets/note_snippets/core_indep.rst b/docs/source/rst_snippets/note_snippets/core_indep.rst index d38d2f9e698..8fe7f93cdf3 100644 --- a/docs/source/rst_snippets/note_snippets/core_indep.rst +++ b/docs/source/rst_snippets/note_snippets/core_indep.rst @@ -1,3 +1,4 @@ .. note:: - All the Core Options are independent options, i.e. They don't depend on other options. + All the Core Options except for ``--additional-license-directory`` are independent options, i.e. + They don't depend on other options. From 7e66c9a98f905c2da845c74a5bc1ba72ce60129f Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 27 Aug 2022 15:23:34 -0700 Subject: [PATCH 13/43] fix docstrings Signed-off-by: Kevin Ji --- src/licensedcode/models.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index 55908468f02..af9debb4388 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -777,8 +777,8 @@ def get_license_dirs( additional_dirs, ): """ - Takes in a list of additional directories specified during license detection - and produces a list of all the subdirectories containing license files. + Returns a list of all subdirectories containing license files within the + input list of additional directories. These directories do not have to be absolute paths. """ # convert to absolute path in case user passes in a relative path, which messes up building rules from licenses return [f"{str(Path(path).absolute())}/licenses" for path in additional_dirs] @@ -788,8 +788,8 @@ def get_rule_dirs( additional_dirs, ): """ - Takes in a list of additional directories specified during license detection - and produces a list of all the subdirectories containing rule files. + Returns a list of all subdirectories containing rule files within the + input list of additional directories. These directories do not have to be absolute paths. """ return [f"{str(Path(path).absolute())}/rules" for path in additional_dirs] @@ -818,8 +818,8 @@ def load_licenses_from_multiple_dirs( with_deprecated=False, ): """ - Takes in a list of directories containing additional licenses to use in - license detection and combines all the licenses into the same mapping. + Combines a list of directories containing additional licenses into the same mapping. + These directory paths do not need to be absolute paths. """ combined_licenses = {} for license_dir in license_directories: @@ -834,9 +834,9 @@ def get_rules_from_multiple_dirs( rule_directories, ): """ - Takes in a license database, which is a mapping from key->License objects, - and a list of all directories containing rules to use in license detection. + Return a mapping of {key: License} built from a list of ``license_directories``. Combines all rules together into the same data structure and validates them. + These license directories do not need to be absolute paths. """ if rule_directories: combined_rules = [] @@ -860,8 +860,8 @@ class InvalidLicense(Exception): def validate_additional_license_data(additional_directories): """ - Takes in directories of additional licenses and determines whether they are valid. - If there are any invalid licenses, raises an exception. + Raises an exception if there are any invalid licenses in the directories of + additional licenses. """ licenses = load_licenses_from_multiple_dirs(additional_directories) errors, _, _ = License.validate( @@ -880,9 +880,8 @@ def validate_additional_license_data(additional_directories): def _ignorable_clue_error(rule): """ - Helper method to validate a single rule's ignorable clues. - Returns a pair of the result and expected ignorable clues if - there is an error. Otherwise, returns None. + Return a pair of the result of validating a rule's ignorable clues and expected ignorable clues + if there is an error. Otherwise, returns None. """ result = get_ignorables(rule.text_file) expected = get_normalized_ignorables(rule) @@ -900,8 +899,8 @@ def _ignorable_clue_error(rule): def validate_ignorable_clues(rule_directories): """ - Validates that all expected ignorable clues declared in a Rule - are properly detected in the rule text file. + Raises an exception if any ignorable clues declared in a Rule are improperly detected + in the rule text file. """ combined_rules = [] for rules_dir in rule_directories: From 9679feb5277b5b3a08eabf65741694dfd786193e Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 27 Aug 2022 16:05:52 -0700 Subject: [PATCH 14/43] refactor API to not use additional_directories except when reindexing Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 61 ------------------------------ src/licensedcode/plugin_license.py | 30 +++++++-------- src/scancode/api.py | 1 - 3 files changed, 14 insertions(+), 78 deletions(-) diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 1b194085e4f..4d7c06b0d7c 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -35,7 +35,6 @@ LICENSE_INDEX_FILENAME = 'index_cache' LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile' LICENSE_CHECKSUM_FILE = 'scancode_license_index_tree_checksums' -CACHED_DIRECTORIES_FILENAME = 'cached_directories' @attr.s(slots=True) @@ -78,7 +77,6 @@ def load_or_build( idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) create_dir(idx_cache_dir) cache_file = os.path.join(idx_cache_dir, LICENSE_INDEX_FILENAME) - cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) has_cache = os.path.exists(cache_file) and os.path.getsize(cache_file) @@ -87,8 +85,6 @@ def load_or_build( try: # save the list of additional directories included in the cache, or None if the cache does not # include any additional directories - with open(cached_directories_file, 'wb') as file: - pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) return load_cache_file(cache_file) except Exception as e: # work around some rare Windows quirks @@ -151,11 +147,6 @@ def load_or_build( with open(cache_file, 'wb') as fn: pickle.dump(license_cache, fn, protocol=PICKLE_PROTOCOL) - # save the list of additional directories included in the cache, or None if the cache does not - # include any additional directories - with open(cached_directories_file, 'wb') as file: - pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) - return license_cache except lockfile.LockTimeout: @@ -376,8 +367,6 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie # additional_directories is originally a tuple additional_directories = list(additional_directories) + get_paths_to_installed_licenses_and_rules() - if need_cache_rebuild(additional_directories): - force = True if force or not _LICENSE_CACHE: _LICENSE_CACHE = LicenseCache.load_or_build( licensedcode_cache_dir=licensedcode_cache_dir, @@ -390,56 +379,6 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie ) -def need_cache_rebuild(additional_directories): - """ - Return true if we need to rebuild the index cache. - """ - idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) - cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) - - has_cached_directories = os.path.exists(cached_directories_file) - should_rebuild_cache = False - - if has_cached_directories: - # if we have cached additional directories of licenses, check if those licenses are equal to the additional - # directories passed in - with open(cached_directories_file, 'rb') as file: - # it's possible that pickle.load(file) results in None - try: - cached_additional_directories = pickle.load(file) - except EOFError: - cached_additional_directories = set() - - # we need to rebuild the cache if the list of additional directories we passed in is not a subset of - # the set of additional directories currently included in the index cache - should_rebuild_cache = additional_directories is not None \ - and not set(additional_directories).issubset(cached_additional_directories) - else: - # otherwise, we don't have a file of cached directories. If there are additional directories passed in, - # we know we need to make a new cache file. - if additional_directories: - should_rebuild_cache = True - return should_rebuild_cache - - -def load_cache_file(cache_file): - """ - Return a LicenseCache loaded from ``cache_file``. - """ - with open(cache_file, 'rb') as lfc: - # Note: weird but read() + loads() is much (twice++???) faster than load() - try: - return pickle.load(lfc) - except Exception as e: - msg = ( - 'ERROR: Failed to load license cache (the file may be corrupted ?).\n' - f'Please delete "{cache_file}" and retry.\n' - 'If the problem persists, copy this error message ' - 'and submit a bug report at https://github.com/nexB/scancode-toolkit/issues/' - ) - raise Exception(msg) from e - - def get_index(force=False, index_all_languages=False, additional_directories=None): """ Return and eventually build and cache a LicenseIndex. diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 1586a6bc21a..54d72a35a02 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -11,6 +11,7 @@ from functools import partial import attr +import click from commoncode.cliutils import MISC_GROUP from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import SCAN_OPTIONS_GROUP @@ -18,7 +19,6 @@ from commoncode.resource import clean_path from plugincode.scan import ScanPlugin from plugincode.scan import scan_impl -import click from scancode.api import SCANCODE_LICENSEDB_URL @@ -56,7 +56,7 @@ def reindex_licenses(ctx, param, value): from licensedcode.cache import get_index import click click.echo('Rebuilding the license index...') - get_index(force=True) + get_index(force=True, additional_directories=ctx.params.get("additional_license_directory")) click.echo('Done.') ctx.exit(0) @@ -140,15 +140,6 @@ class LicenseScanner(ScanPlugin): help_group=SCAN_OPTIONS_GROUP, ), - PluggableCommandLineOption( - ('--external-license-directory',), - required_options=['license'], - multiple=True, - type=click.Path(exists=True, readable=True, path_type=str), - help='Include additional directories for license detection.', - help_group=SCAN_OPTIONS_GROUP, - ), - PluggableCommandLineOption( ('--reindex-licenses',), is_flag=True, is_eager=True, @@ -164,8 +155,18 @@ class LicenseScanner(ScanPlugin): help='[EXPERIMENTAL] Rebuild the license index including texts all ' 'languages (and not only English) and exit.', help_group=MISC_GROUP, - ) + ), + PluggableCommandLineOption( + ('--additional-license-directory',), + required_options=['reindex_licenses'], + multiple=True, + type=click.Path(exists=True, readable=True, file_okay=False, resolve_path=True, path_type=str), + metavar='DIR', + help='Include this directory with additional custom licenses and license rules ' + 'in the license detection index.', + help_group=MISC_GROUP, + ), ] def is_enabled(self, license, **kwargs): # NOQA @@ -177,8 +178,7 @@ def setup(self, **kwargs): loaded index. """ from licensedcode.cache import populate_cache - additional_directories = kwargs.get('external_license_directory') - populate_cache(additional_directories=additional_directories) + populate_cache() def get_scanner( self, @@ -187,7 +187,6 @@ def get_scanner( license_text_diagnostics=False, license_url_template=SCANCODE_LICENSEDB_URL, unknown_licenses=False, - additional_directories=None, **kwargs ): @@ -198,7 +197,6 @@ def get_scanner( license_text_diagnostics=license_text_diagnostics, license_url_template=license_url_template, unknown_licenses=unknown_licenses, - additional_directories=additional_directories, ) def process_codebase(self, codebase, unknown_licenses, **kwargs): diff --git a/src/scancode/api.py b/src/scancode/api.py index 1acd26ef174..2df62a2a4c5 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -142,7 +142,6 @@ def get_licenses( license_url_template=SCANCODE_LICENSEDB_URL, unknown_licenses=False, deadline=sys.maxsize, - additional_directories=None, **kwargs, ): """ From 6aae7e26ced612a2b03623dd83fdc1f2f32377f7 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 27 Aug 2022 16:11:01 -0700 Subject: [PATCH 15/43] Always consider multiple directories when generating index Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 42 +++++++++++++++------------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 4d7c06b0d7c..ab425c4aa6c 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -95,7 +95,6 @@ def load_or_build( from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd - from licensedcode.models import load_licenses from licensedcode.models import load_licenses_from_multiple_dirs from licensedcode.models import get_license_dirs from licensedcode.models import validate_additional_license_data @@ -113,13 +112,10 @@ def load_or_build( # Here, the cache is either stale or non-existing: we need to # rebuild all cached data (e.g. mostly the index) and cache it - if additional_directories: - additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) - validate_additional_license_data(additional_license_dirs) - combined_directories = [licenses_data_dir] + additional_license_dirs - licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories) - else: - licenses_db = load_licenses(licenses_data_dir=licenses_data_dir) + additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) + validate_additional_license_data(additional_license_dirs) + combined_directories = [licenses_data_dir] + additional_license_dirs + licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories) # create a single merged index containing license data from licenses_data_dir # and data from additional directories @@ -187,24 +183,18 @@ def build_index( rules_data_dir = rules_data_dir or rdd if not licenses_db: - if additional_directories: - # combine the licenses in these additional directories with the licenses in the original DB - additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) - combined_license_directories = [licenses_data_dir] + additional_license_dirs - # generate a single combined license db with all licenses - licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories) - else: - licenses_db = load_licenses(licenses_data_dir=licenses_data_dir) - - if additional_directories: - # if we have additional directories, extract the rules from them - additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) - validate_ignorable_clues(additional_rule_dirs) - # then combine the rules in these additional directories with the rules in the original rules directory - combined_rule_directories = [rules_data_dir] + additional_rule_dirs - rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories) - else: - rules = get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) + # combine the licenses in these additional directories with the licenses in the original DB + additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) + combined_license_directories = [licenses_data_dir] + additional_license_dirs + # generate a single combined license db with all licenses + licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories) + + # if we have additional directories, extract the rules from them + additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) + validate_ignorable_clues(additional_rule_dirs) + # then combine the rules in these additional directories with the rules in the original rules directory + combined_rule_directories = [rules_data_dir] + additional_rule_dirs + rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories) legalese = common_license_words spdx_tokens = set(get_all_spdx_key_tokens(licenses_db)) From f13ed1d50337703af8596d9ea4333621597ae6b7 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sat, 27 Aug 2022 16:24:26 -0700 Subject: [PATCH 16/43] Ensure licenses are unique when loading licenses from multiple directories Signed-off-by: Kevin Ji --- src/licensedcode/models.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index af9debb4388..de29e38fd97 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -822,10 +822,20 @@ def load_licenses_from_multiple_dirs( These directory paths do not need to be absolute paths. """ combined_licenses = {} + combined_licenses_keys = set() for license_dir in license_directories: licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False) - # this syntax for merging is described here: https://stackoverflow.com/a/26853961 - combined_licenses = {**combined_licenses, **licenses} + # check if two dictionaries have duplicate keys + licenses_keys = set(licenses.keys()) + duplicate_keys = combined_licenses_keys.intersection(licenses_keys) + if duplicate_keys: + message = ['Duplicate licenses found when loading additional licenses.'] + message.append(', '.join(duplicate_keys)) + raise ValueError('\n'.join(message)) + combined_licenses.update(licenses) + # we keep combined_licenses_keys and licenses_keys as separate variables so we can avoid calling set() + # on the dictionary keys every single iteration + combined_licenses_keys.update(licenses_keys) return combined_licenses From 9b03eea1dee2d3f40d0a2c7268b2374729f8eb38 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:32:01 -0700 Subject: [PATCH 17/43] add callback for --additional-license-directory and include additional directories in scancode output Signed-off-by: Kevin Ji --- docs/source/cli-reference/core-options.rst | 9 +- src/licensedcode/cache.py | 42 ++- src/licensedcode/plugin_license.py | 20 +- src/scancode/api.py | 2 +- src/scancode/cli.py | 9 + .../licenses_to_install1/gpl-1.0.LICENSE | 254 ------------------ .../licenses_to_install1/setup.cfg | 2 +- .../licenses_to_install1/test.LICENSE | 1 + .../tests/data/example-installed-1.txt | 6 +- .../external_licenses/scan.expected.json | 42 ++- .../scan_multiple.expected.json | 45 +++- .../installed_licenses/scan.expected.json | 37 ++- tests/licensedcode/test_plugin_license.py | 18 +- 13 files changed, 173 insertions(+), 314 deletions(-) delete mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE create mode 100644 tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE diff --git a/docs/source/cli-reference/core-options.rst b/docs/source/cli-reference/core-options.rst index f9fdf7e6bbe..b73d527066e 100644 --- a/docs/source/cli-reference/core-options.rst +++ b/docs/source/cli-reference/core-options.rst @@ -114,13 +114,14 @@ Comparing Progress Message Options ├── privateLicense2.RULE └── privateLicense2.yml - Here is an example of reindexing the license cache using the ``--additional-license-directory PATH`` option with a single directory:: + Here is an example of reindexing the license cache using the ``--additional-license-directory PATH`` option with a single directory. + Note that ``--reindex-licenses`` **must** come after ``--additional-license-directory``:: - scancode --reindex-licenses --additional-license-directory /home/user/external_licenses/license1 + scancode --additional-license-directory /home/user/external_licenses/license1 --reindex-licenses You can also include multiple directories like so:: - scancode --reindex-licenses --additional-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external2 + scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external2 --reindex-licenses If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, you can simply run scans after reindexing with those directories and they will be included. :: @@ -130,7 +131,7 @@ Comparing Progress Message Options However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` and ``home/user/external_licenses/external3``, you would need to reindex the license index with those directories as parameters. :: - scancode --additional-license-directory /home/user/external_licenses/external1 --external-license-directory /home/user/external_licenses/external3 + scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external3 --reindex-licenses .. diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index ab425c4aa6c..fc9a13d9b75 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -35,6 +35,7 @@ LICENSE_INDEX_FILENAME = 'index_cache' LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile' LICENSE_CHECKSUM_FILE = 'scancode_license_index_tree_checksums' +CACHED_DIRECTORIES_FILENAME = 'cached_directories' @attr.s(slots=True) @@ -98,6 +99,7 @@ def load_or_build( from licensedcode.models import load_licenses_from_multiple_dirs from licensedcode.models import get_license_dirs from licensedcode.models import validate_additional_license_data + from licensedcode.models import get_paths_to_installed_licenses_and_rules from scancode import lockfile licenses_data_dir = licenses_data_dir or ldd @@ -112,6 +114,20 @@ def load_or_build( # Here, the cache is either stale or non-existing: we need to # rebuild all cached data (e.g. mostly the index) and cache it + # include installed licenses + if not additional_directories: + additional_directories = get_paths_to_installed_licenses_and_rules() + else: + # additional_directories is originally a tuple + additional_directories = list(additional_directories) + get_paths_to_installed_licenses_and_rules() + + # persist additional directories as a file so that we can include it in the scancode output as extra info + # only persist when we're generating a new license cache + idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) + cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) + with open(cached_directories_file, 'wb') as file: + pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) + additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) validate_additional_license_data(additional_license_dirs) combined_directories = [licenses_data_dir] + additional_license_dirs @@ -347,16 +363,8 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie """ Load or build and cache a LicenseCache. Return None. """ - from licensedcode.models import get_paths_to_installed_licenses_and_rules global _LICENSE_CACHE - # include installed licenses - if not additional_directories: - additional_directories = get_paths_to_installed_licenses_and_rules() - else: - # additional_directories is originally a tuple - additional_directories = list(additional_directories) + get_paths_to_installed_licenses_and_rules() - if force or not _LICENSE_CACHE: _LICENSE_CACHE = LicenseCache.load_or_build( licensedcode_cache_dir=licensedcode_cache_dir, @@ -369,6 +377,24 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie ) +def load_cache_file(cache_file): + """ + Return a LicenseCache loaded from ``cache_file``. + """ + with open(cache_file, 'rb') as lfc: + # Note: weird but read() + loads() is much (twice++???) faster than load() + try: + return pickle.load(lfc) + except Exception as e: + msg = ( + 'ERROR: Failed to load license cache (the file may be corrupted ?).\n' + f'Please delete "{cache_file}" and retry.\n' + 'If the problem persists, copy this error message ' + 'and submit a bug report at https://github.com/nexB/scancode-toolkit/issues/' + ) + raise Exception(msg) from e + + def get_index(force=False, index_all_languages=False, additional_directories=None): """ Return and eventually build and cache a LicenseIndex. diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 54d72a35a02..528276c7f44 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -56,11 +56,27 @@ def reindex_licenses(ctx, param, value): from licensedcode.cache import get_index import click click.echo('Rebuilding the license index...') - get_index(force=True, additional_directories=ctx.params.get("additional_license_directory")) + get_index(force=True) click.echo('Done.') ctx.exit(0) +def reindex_licenses_additional(ctx, param, value): + """ + Rebuild and cache the license index using additional directories passed in + from command line. + """ + if not value or ctx.resilient_parsing: + return + + # TODO: check for temp file configuration and use that for the cache!!! + from licensedcode.cache import get_index + import click + click.echo('Rebuilding the license index with additional directories...') + get_index(force=True, additional_directories=value) + click.echo('Done.') + ctx.exit(0) + def reindex_licenses_all_languages(ctx, param, value): """ EXPERIMENTAL: Rebuild and cache the license index including all languages @@ -163,9 +179,11 @@ class LicenseScanner(ScanPlugin): multiple=True, type=click.Path(exists=True, readable=True, file_okay=False, resolve_path=True, path_type=str), metavar='DIR', + callback=reindex_licenses_additional, help='Include this directory with additional custom licenses and license rules ' 'in the license detection index.', help_group=MISC_GROUP, + is_eager=True, ), ] diff --git a/src/scancode/api.py b/src/scancode/api.py index 2df62a2a4c5..dc17f4e5320 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -168,7 +168,7 @@ def get_licenses( from licensedcode import cache from licensedcode.spans import Span - idx = cache.get_index(additional_directories=additional_directories) + idx = cache.get_index() detected_licenses = [] detected_expressions = [] diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 9b4cb0a78b2..1f8572dcc49 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -861,6 +861,15 @@ def echo_func(*_args, **_kwargs): cle.options = pretty_params or {} # useful for debugging cle.extra_data['system_environment'] = system_environment = {} + + from licensedcode.cache import CACHED_DIRECTORIES_FILENAME, LICENSE_INDEX_DIR + import pickle + idx_cache_dir = os.path.join(scancode_config.licensedcode_cache_dir, LICENSE_INDEX_DIR) + cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) + with open(cached_directories_file, 'rb') as file: + cached_additional_directories = pickle.load(file) + cle.extra_data['additional_directories'] = cached_additional_directories + system_environment['operating_system'] = commoncode.system.current_os system_environment['cpu_architecture'] = commoncode.system.current_arch system_environment['platform'] = platform.platform() diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE deleted file mode 100644 index c510b2875fe..00000000000 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/gpl-1.0.LICENSE +++ /dev/null @@ -1,254 +0,0 @@ -GNU GENERAL PUBLIC LICENSE - Version 1, February 1989 - - Copyright (C) 1989 Free Software Foundation, Inc. - 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The license agreements of most software companies try to keep users -at the mercy of those companies. By contrast, our General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. The -General Public License applies to the Free Software Foundation's -software and to any other program whose authors commit to using it. -You can use it for your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Specifically, the General Public License is designed to make -sure that you have the freedom to give away or sell copies of free -software, that you receive source code or can get it if you want it, -that you can change the software or use pieces of it in new free -programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of a such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must tell them their rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - The precise terms and conditions for copying, distribution and -modification follow. - - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any program or other work which -contains a notice placed by the copyright holder saying it may be -distributed under the terms of this General Public License. The -"Program", below, refers to any such program or work, and a "work based -on the Program" means either the Program or any work containing the -Program or a portion of it, either verbatim or with modifications. Each -licensee is addressed as "you". - - 1. You may copy and distribute verbatim copies of the Program's source -code as you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this -General Public License and to the absence of any warranty; and give any -other recipients of the Program a copy of this General Public License -along with the Program. You may charge a fee for the physical act of -transferring a copy. - - 2. You may modify your copy or copies of the Program or any portion of -it, and copy and distribute such modifications under the terms of Paragraph -1 above, provided that you also do the following: - - a) cause the modified files to carry prominent notices stating that - you changed the files and the date of any change; and - - b) cause the whole of any work that you distribute or publish, that - in whole or in part contains the Program or any part thereof, either - with or without modifications, to be licensed at no charge to all - third parties under the terms of this General Public License (except - that you may choose to grant warranty protection to some or all - third parties, at your option). - - c) If the modified program normally reads commands interactively when - run, you must cause it, when started running for such interactive use - in the simplest and most usual way, to print or display an - announcement including an appropriate copyright notice and a notice - that there is no warranty (or else, saying that you provide a - warranty) and that users may redistribute the program under these - conditions, and telling the user how to view a copy of this General - Public License. - - d) You may charge a fee for the physical act of transferring a - copy, and you may at your option offer warranty protection in - exchange for a fee. - -Mere aggregation of another independent work with the Program (or its -derivative) on a volume of a storage or distribution medium does not bring -the other work under the scope of these terms. - - - 3. You may copy and distribute the Program (or a portion or derivative of -it, under Paragraph 2) in object code or executable form under the terms of -Paragraphs 1 and 2 above provided that you also do one of the following: - - a) accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of - Paragraphs 1 and 2 above; or, - - b) accompany it with a written offer, valid for at least three - years, to give any third party free (except for a nominal charge - for the cost of distribution) a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of - Paragraphs 1 and 2 above; or, - - c) accompany it with the information you received as to where the - corresponding source code may be obtained. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form alone.) - -Source code for a work means the preferred form of the work for making -modifications to it. For an executable file, complete source code means -all the source code for all modules it contains; but, as a special -exception, it need not include source code for modules which are standard -libraries that accompany the operating system on which the executable -file runs, or for standard header files or definitions files that -accompany that operating system. - - 4. You may not copy, modify, sublicense, distribute or transfer the -Program except as expressly provided under this General Public License. -Any attempt otherwise to copy, modify, sublicense, distribute or transfer -the Program is void, and will automatically terminate your rights to use -the Program under this License. However, parties who have received -copies, or rights to use copies, from you under this General Public -License will not have their licenses terminated so long as such parties -remain in full compliance. - - 5. By copying, distributing or modifying the Program (or any work based -on the Program) you indicate your acceptance of this license to do so, -and all its terms and conditions. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the original -licensor to copy, distribute or modify the Program subject to these -terms and conditions. You may not impose any further restrictions on the -recipients' exercise of the rights granted herein. - - - 7. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of the license which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -the license, you may choose any version ever published by the Free Software -Foundation. - - 8. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - - Appendix: How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to humanity, the best way to achieve this is to make it -free software which everyone can redistribute and change under these -terms. - - To do so, attach the following notices to the program. It is safest to -attach them to the start of each source file to most effectively convey -the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - - Copyright (C) 19yy - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 1, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) 19xx name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the -appropriate parts of the General Public License. Of course, the -commands you use may be called something other than `show w' and `show -c'; they could even be mouse-clicks or menu items--whatever suits your -program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - program `Gnomovision' (a program to direct compilers to make passes - at assemblers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -That's all there is to it! \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg index b463cb6c31e..f5483e8ca2e 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg @@ -1,6 +1,6 @@ [metadata] license_files = - gpl-1.0.LICENSE + test.LICENSE [aliases] release = clean --all bdist_wheel --plat-name manylinux1_x86_64 --python-tag py3 \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE new file mode 100644 index 00000000000..6c7f04de819 --- /dev/null +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE @@ -0,0 +1 @@ +This is a test license. That's all there is to it! \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt index 932b6e6357f..c4d8e86562e 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt @@ -1,8 +1,4 @@ This is a test license that must be installed into ScanCode Toolkit. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor -incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud -exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure -dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. -Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit -anim id est laborum. +incididunt ut labore et dolore magna aliqua. diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json index abb56159e26..07acc9b81e2 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -2,15 +2,19 @@ "headers": [ { "tool_name": "scancode-toolkit", + "tool_version": "31.0.0rc1", "options": { - "input": "", - "--external-license-directory": "", - "--json": "", - "--license": true, - "--strip-root": true + "input": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/external_licenses/scan" + ], + "--json-pp": "../temp.json", + "--license": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2022-08-29T004616.713244", + "end_timestamp": "2022-08-29T004618.624574", "output_format_version": "2.0.0", + "duration": 1.911355972290039, "message": null, "errors": [], "warnings": [], @@ -18,18 +22,30 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", - "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", - "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", + "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", + "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" }, - "spdx_license_list_version": "3.16", - "files_count": 2 + "additional_directories": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", + "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" + ], + "spdx_license_list_version": "3.17", + "files_count": 1 } } ], "files": [ { - "path": "license.txt", + "path": "scan", + "type": "directory", + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "scan_errors": [] + }, + { + "path": "scan/license.txt", "type": "file", "licenses": [ { @@ -41,7 +57,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", + "homepage_url": null, "text_url": "", "reference_url": "https://scancode-licensedb.aboutcode.org/example1", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", @@ -78,4 +94,4 @@ "scan_errors": [] } ] -} +} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index d5f19b0e8f8..5568f173d84 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -2,15 +2,19 @@ "headers": [ { "tool_name": "scancode-toolkit", + "tool_version": "31.0.0rc1", "options": { - "input": "", - "--external-license-directory": "", - "--json": "", - "--license": true, - "--strip-root": true + "input": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/external_licenses/scan" + ], + "--json-pp": "../temp.json", + "--license": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2022-08-29T004949.561867", + "end_timestamp": "2022-08-29T004951.423696", "output_format_version": "2.0.0", + "duration": 1.861854076385498, "message": null, "errors": [], "warnings": [], @@ -18,18 +22,31 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", - "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", - "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", + "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", + "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" }, - "spdx_license_list_version": "3.16", - "files_count": 2 + "additional_directories": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example2", + "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" + ], + "spdx_license_list_version": "3.17", + "files_count": 1 } } ], "files": [ { - "path": "license.txt", + "path": "scan", + "type": "directory", + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "scan_errors": [] + }, + { + "path": "scan/license.txt", "type": "file", "licenses": [ { @@ -41,7 +58,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", + "homepage_url": null, "text_url": "", "reference_url": "https://scancode-licensedb.aboutcode.org/example1", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", @@ -79,7 +96,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", + "homepage_url": null, "text_url": "", "reference_url": "https://scancode-licensedb.aboutcode.org/example2", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example2.LICENSE", @@ -117,4 +134,4 @@ "scan_errors": [] } ] -} +} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index ad21fd701ca..a91be77b2a4 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -2,14 +2,19 @@ "headers": [ { "tool_name": "scancode-toolkit", + "tool_version": "31.0.0rc1", "options": { - "input": "", - "--json": "", - "--license": true, - "--strip-root": true + "input": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/installed_licenses/scan" + ], + "--json-pp": "../temp.json", + "--license": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2022-08-29T004350.528873", + "end_timestamp": "2022-08-29T004352.269691", "output_format_version": "2.0.0", + "duration": 1.7408447265625, "message": null, "errors": [], "warnings": [], @@ -17,10 +22,14 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", - "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", - "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" + "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", + "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", + "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" }, + "additional_directories": [ + "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", + "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" + ], "spdx_license_list_version": "3.17", "files_count": 1 } @@ -28,7 +37,15 @@ ], "files": [ { - "path": "license.txt", + "path": "scan", + "type": "directory", + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "scan_errors": [] + }, + { + "path": "scan/license.txt", "type": "file", "licenses": [ { @@ -40,7 +57,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", + "homepage_url": null, "text_url": "", "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", @@ -77,4 +94,4 @@ "scan_errors": [] } ] -} +} \ No newline at end of file diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index d5ca0a6c1d2..5248d8d488f 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -247,11 +247,17 @@ def test_detection_with_single_external_license_directory(): test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) example1_dir = test_env.get_test_loc('example_external_licenses/example1') result_file = test_env.get_temp_file('json') + + # first reindex with the example directory + run_scan_click([ + '--additional-license-directory', example1_dir, + '--reindex-licenses', + ]) + args = [ '--license', '--strip-root', '--verbose', - '--external-license-directory', example1_dir, '--json', result_file, test_dir, ] @@ -266,12 +272,18 @@ def test_detection_with_multiple_external_license_directories(): example1_dir = test_env.get_test_loc('example_external_licenses/example1') example2_dir = test_env.get_test_loc('example_external_licenses/example2') result_file = test_env.get_temp_file('json') + + # first reindex with the example directories + run_scan_click([ + '--additional-license-directory', example1_dir, + '--additional-license-directory', example2_dir, + '--reindex-licenses', + ]) + args = [ '--license', '--strip-root', '--verbose', - '--external-license-directory', example1_dir, - '--external-license-directory', example2_dir, '--json', result_file, test_dir, ] From 40f3be97aa32aa3a8640cc26b773733a7f86e115 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:33:36 -0700 Subject: [PATCH 18/43] fix help.txt to include --additional-license-directory Signed-off-by: Kevin Ji --- tests/scancode/data/help/help.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scancode/data/help/help.txt b/tests/scancode/data/help/help.txt index d03c64a3c3a..160e60e93bc 100644 --- a/tests/scancode/data/help/help.txt +++ b/tests/scancode/data/help/help.txt @@ -158,6 +158,10 @@ Options: scan depth limit. miscellaneous: + --additional-license-directory DIR + Include this directory with additional + custom licenses and license rules in the + license detection index. --reindex-licenses Rebuild the license index and exit. --reindex-licenses-for-all-languages [EXPERIMENTAL] Rebuild the license index From 983024eb41354f1f9828bb1a39aacbf779d282b4 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:39:39 -0700 Subject: [PATCH 19/43] fix docs Signed-off-by: Kevin Ji --- docs/source/rst_snippets/basic_options.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/rst_snippets/basic_options.rst b/docs/source/rst_snippets/basic_options.rst index d5394b3fda9..1b7b8ae885c 100644 --- a/docs/source/rst_snippets/basic_options.rst +++ b/docs/source/rst_snippets/basic_options.rst @@ -55,8 +55,6 @@ documenting a program's options. For example: .. include:: /rst_snippets/note_snippets/basic_clpieu.rst -` - --generated Classify automatically generated code files with a flag. --max-email INT Report only up to INT emails found in a From c8391d00c5807d1c3ce2446da84d30f594bf0b12 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:44:06 -0700 Subject: [PATCH 20/43] fix basic-options.rst Signed-off-by: Kevin Ji --- docs/source/cli-reference/basic-options.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/cli-reference/basic-options.rst b/docs/source/cli-reference/basic-options.rst index 561e4659723..70fbec2c1c3 100644 --- a/docs/source/cli-reference/basic-options.rst +++ b/docs/source/cli-reference/basic-options.rst @@ -7,10 +7,6 @@ .. include:: /rst_snippets/note_snippets/synopsis_install_quickstart.rst ----- - - - ---- ``--generated`` Options From 1bc43afe08873ab9faf9fba9e0ef2dabd12b38ad Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:46:53 -0700 Subject: [PATCH 21/43] add check in cli.py to see if cached directories file actually exists Signed-off-by: Kevin Ji --- src/scancode/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 1f8572dcc49..188af532e94 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -866,8 +866,10 @@ def echo_func(*_args, **_kwargs): import pickle idx_cache_dir = os.path.join(scancode_config.licensedcode_cache_dir, LICENSE_INDEX_DIR) cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) - with open(cached_directories_file, 'rb') as file: - cached_additional_directories = pickle.load(file) + cached_additional_directories = {} + if os.path.exists(cached_directories_file): + with open(cached_directories_file, 'rb') as file: + cached_additional_directories = pickle.load(file) cle.extra_data['additional_directories'] = cached_additional_directories system_environment['operating_system'] = commoncode.system.current_os From fc7b967aa1aa9161da82d05baac7f2514a0c7007 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:58:12 -0700 Subject: [PATCH 22/43] fix expected test results directory path Signed-off-by: Kevin Ji --- .../data/plugin_license/external_licenses/scan.expected.json | 2 +- .../external_licenses/scan_multiple.expected.json | 2 +- .../data/plugin_license/installed_licenses/scan.expected.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json index 07acc9b81e2..3f14be49d85 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -45,7 +45,7 @@ "scan_errors": [] }, { - "path": "scan/license.txt", + "path": "license.txt", "type": "file", "licenses": [ { diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index 5568f173d84..74446633d7b 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -46,7 +46,7 @@ "scan_errors": [] }, { - "path": "scan/license.txt", + "path": "license.txt", "type": "file", "licenses": [ { diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index a91be77b2a4..8452b790e5b 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -45,7 +45,7 @@ "scan_errors": [] }, { - "path": "scan/license.txt", + "path": "license.txt", "type": "file", "licenses": [ { From 597c616e3c0077ef0de5608f4da08df2b1b146b4 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 21:59:27 -0700 Subject: [PATCH 23/43] fix underline in docs Signed-off-by: Kevin Ji --- docs/source/cli-reference/core-options.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/cli-reference/core-options.rst b/docs/source/cli-reference/core-options.rst index b73d527066e..7bfc84e86b6 100644 --- a/docs/source/cli-reference/core-options.rst +++ b/docs/source/cli-reference/core-options.rst @@ -83,8 +83,8 @@ Comparing Progress Message Options .. [ToDo] Research and Write Better -``--additional-license-directory`` Options ----------------------------------------- +``--additional-license-directory`` Option +----------------------------------------- .. admonition:: Dependency From 85001c1bf457004018757e82560847d1dbca6d84 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 22:08:23 -0700 Subject: [PATCH 24/43] fix expected results for external and installed license tests Signed-off-by: Kevin Ji --- .../plugin_license/external_licenses/scan.expected.json | 8 -------- .../external_licenses/scan_multiple.expected.json | 8 -------- .../plugin_license/installed_licenses/scan.expected.json | 8 -------- 3 files changed, 24 deletions(-) diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json index 3f14be49d85..f50623609a6 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -36,14 +36,6 @@ } ], "files": [ - { - "path": "scan", - "type": "directory", - "licenses": [], - "license_expressions": [], - "percentage_of_license_text": 0, - "scan_errors": [] - }, { "path": "license.txt", "type": "file", diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index 74446633d7b..b4fce4d33a0 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -37,14 +37,6 @@ } ], "files": [ - { - "path": "scan", - "type": "directory", - "licenses": [], - "license_expressions": [], - "percentage_of_license_text": 0, - "scan_errors": [] - }, { "path": "license.txt", "type": "file", diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index 8452b790e5b..483f64fb391 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -36,14 +36,6 @@ } ], "files": [ - { - "path": "scan", - "type": "directory", - "licenses": [], - "license_expressions": [], - "percentage_of_license_text": 0, - "scan_errors": [] - }, { "path": "license.txt", "type": "file", From 74970096d56e592602aa41e56e62a0ed40bf08d6 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 22:08:47 -0700 Subject: [PATCH 25/43] put license installation into posix azure pipeline Signed-off-by: Kevin Ji --- azure-pipelines.yml | 21 ++++----------------- etc/ci/azure-posix.yml | 6 ++++++ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6cd7936fef5..72920ce23c9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -79,6 +79,10 @@ jobs: venv/bin/pytest -n 3 -vvs --test-suite=all \ tests/licensedcode/test_zzzz_cache.py + installed_licenses: | + venv/bin/pytest -vvs --test-suite=all \ + tests/licensedcode/test_installed_license_detection.py + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython @@ -169,23 +173,6 @@ jobs: test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - template: etc/ci/azure-posix-installed-external-license.yml - parameters: - job_name: installed_external_license_ubuntu - image_name: ubuntu-20.04 - python_versions: ['3.8'] - - - template: etc/ci/azure-win-installed-external-license.yml - parameters: - job_name: installed_external_license_windows - image_name: windows-2022 - python_versions: ['3.8'] - - - template: etc/ci/azure-posix-installed-external-license.yml - parameters: - job_name: installed_external_license_macos11 - image_name: macos-11 - python_versions: ['3.8'] ################################################################################ # Test using many version of Click to work around any regressions in their API diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 9fdc7f1522e..2f52a370ca4 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -4,6 +4,9 @@ parameters: python_versions: [] test_suites: {} python_architecture: x64 + install_packages: | + venv/bin/python3 -m pip install --upgrade pip + venv/bin/python3 -m pip install tests/licensedcode/data/example_external_licenses/licenses_to_install1 jobs: - job: ${{ parameters.job_name }} @@ -35,5 +38,8 @@ jobs: ./configure --clean && ./configure --dev displayName: '${{ pyver }} - Configure' + - script: ${{ parameters.install_packages }} + displayName: '${{ pyver }} - Install license' + - script: $(test_suite) displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' From d6068c64918db51a76802081939dd9152f67ef3d Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 22:20:07 -0700 Subject: [PATCH 26/43] remove setuptools and wheel from setup.py Signed-off-by: Kevin Ji --- .../example_external_licenses/licenses_to_install1/setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py index d947bf9f3b7..bc713a11e73 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py @@ -39,8 +39,6 @@ ], install_requires=[ 'scancode-toolkit', - 'wheel', - 'pytest', ], entry_points={ 'scancode_location_provider': [ From 16513ff40eadd517dca2af6ba090bcbac0a90c6a Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 28 Aug 2022 22:49:49 -0700 Subject: [PATCH 27/43] change from scan to reindex licenses in license library validation test Signed-off-by: Kevin Ji --- tests/licensedcode/test_plugin_license.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 5248d8d488f..6de55f27b28 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -328,20 +328,11 @@ def test_detection_is_correct_in_legacy_npm_package_json(): def test_validate_license_library_returns_errors(): - test_dir = test_env.get_test_loc('plugin_license/license-expression/scan', copy=True) + from licensedcode.models import InvalidLicense licenses_dir = test_env.get_test_loc('plugin_license/validate_licenses') - result_file = test_env.get_temp_file('json') args = [ - '--license', - '--strip-root', - '--verbose', - '--json', result_file, - '--external-license-directory', licenses_dir, - test_dir, + '--additional-license-directory', licenses_dir, + '--reindex-licenses', ] - # the actual error is an InvalidLicense, but run_scan_click has an assert so it - # raises an AssertionError when the code itself throws an InvalidLicense error. - # This is why we assert the test raises an AssertionError but check that the - # error text is an InvalidLicense. - with pytest.raises(AssertionError, match=r"InvalidLicense"): + with pytest.raises(InvalidLicense): run_scan_click(args) From 17df9d0cfbfd0c655df2b51042aabbb383f1fee5 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 4 Sep 2022 15:49:52 -0700 Subject: [PATCH 28/43] Add is_builtin field to Licenses and Rules and modify url output Signed-off-by: Kevin Ji --- src/licensedcode/cache.py | 15 ++++-- src/licensedcode/models.py | 46 ++++++++++++++----- src/scancode/api.py | 20 +++++--- .../external_licenses/scan.expected.json | 6 +-- .../scan_multiple.expected.json | 6 +-- .../installed_licenses/scan.expected.json | 7 ++- .../installed_rules/scan.expected.json | 9 ++-- 7 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index fc9a13d9b75..b45647a73ef 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -129,9 +129,12 @@ def load_or_build( pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) - validate_additional_license_data(additional_license_dirs) + validate_additional_license_data(additional_directories=additional_license_dirs, scancode_license_dir=licenses_data_dir) combined_directories = [licenses_data_dir] + additional_license_dirs - licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories) + licenses_db = load_licenses_from_multiple_dirs( + license_directories=combined_directories, + scancode_license_dir=licenses_data_dir + ) # create a single merged index containing license data from licenses_data_dir # and data from additional directories @@ -207,10 +210,14 @@ def build_index( # if we have additional directories, extract the rules from them additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) - validate_ignorable_clues(additional_rule_dirs) + validate_ignorable_clues(rule_directories=additional_rule_dirs, is_builtin=False) # then combine the rules in these additional directories with the rules in the original rules directory combined_rule_directories = [rules_data_dir] + additional_rule_dirs - rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories) + rules = get_rules_from_multiple_dirs( + licenses_db=licenses_db, + rule_directories=combined_rule_directories, + scancode_rules_dir=rules_data_dir + ) legalese = common_license_words spdx_tokens = set(get_all_spdx_key_tokens(licenses_db)) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index de29e38fd97..c679973108c 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -189,6 +189,13 @@ class License: help='Free text notes.') ) + is_builtin = attr.ib( + default=True, + repr=False, + metadata=dict( + help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.') + ) + # TODO: add the license key(s) this exception applies to is_exception = attr.ib( default=False, @@ -349,12 +356,12 @@ class License: ) @classmethod - def from_dir(cls, key, licenses_data_dir=licenses_data_dir): + def from_dir(cls, key, licenses_data_dir=licenses_data_dir, is_builtin=True): """ Return a new License object for a license ``key`` and load its attribute from a data file stored in ``licenses_data_dir``. """ - lic = cls(key=key) + lic = cls(key=key, is_builtin=is_builtin) data_file = lic.data_file(licenses_data_dir=licenses_data_dir) if exists(data_file): text_file = lic.text_file(licenses_data_dir=licenses_data_dir) @@ -685,6 +692,7 @@ def load_licenses( licenses_data_dir=licenses_data_dir, with_deprecated=False, check_dangling=True, + is_builtin = True, ): """ Return a mapping of {key: License} loaded from license data and text files @@ -712,7 +720,7 @@ def load_licenses( key = file_base_name(data_file) try: - lic = License.from_dir(key=key, licenses_data_dir=licenses_data_dir) + lic = License.from_dir(key=key, licenses_data_dir=licenses_data_dir, is_builtin=is_builtin) except Exception as e: raise Exception(f'Failed to load license: {key} from: file://{licenses_data_dir}/{key}.yml with error: {e}') from e @@ -752,6 +760,7 @@ def get_rules( licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir, validate=False, + is_builtin=True, ): """ Yield Rule objects loaded from a ``licenses_db`` and license files found in @@ -764,6 +773,7 @@ def get_rules( rules = list(load_rules( rules_data_dir=rules_data_dir, + is_builtin=is_builtin, )) if validate: @@ -815,6 +825,7 @@ def get_paths_to_installed_licenses_and_rules(): def load_licenses_from_multiple_dirs( license_directories, + scancode_license_dir, with_deprecated=False, ): """ @@ -824,7 +835,8 @@ def load_licenses_from_multiple_dirs( combined_licenses = {} combined_licenses_keys = set() for license_dir in license_directories: - licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False) + is_builtin = scancode_license_dir == license_dir + licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False, is_builtin=is_builtin) # check if two dictionaries have duplicate keys licenses_keys = set(licenses.keys()) duplicate_keys = combined_licenses_keys.intersection(licenses_keys) @@ -842,6 +854,7 @@ def load_licenses_from_multiple_dirs( def get_rules_from_multiple_dirs( licenses_db, rule_directories, + scancode_rules_dir, ): """ Return a mapping of {key: License} built from a list of ``license_directories``. @@ -851,8 +864,10 @@ def get_rules_from_multiple_dirs( if rule_directories: combined_rules = [] for rules_dir in rule_directories: + is_builtin = rules_dir == scancode_rules_dir r = list(load_rules( rules_data_dir=rules_dir, + is_builtin=is_builtin, )) combined_rules.append(r) # flatten lists of rules into a single iterable @@ -868,12 +883,12 @@ class InvalidLicense(Exception): pass -def validate_additional_license_data(additional_directories): +def validate_additional_license_data(additional_directories, scancode_license_dir): """ Raises an exception if there are any invalid licenses in the directories of additional licenses. """ - licenses = load_licenses_from_multiple_dirs(additional_directories) + licenses = load_licenses_from_multiple_dirs(additional_directories, scancode_license_dir) errors, _, _ = License.validate( licenses, verbose=False, @@ -907,7 +922,7 @@ def _ignorable_clue_error(rule): return result, expected -def validate_ignorable_clues(rule_directories): +def validate_ignorable_clues(rule_directories, is_builtin): """ Raises an exception if any ignorable clues declared in a Rule are improperly detected in the rule text file. @@ -916,6 +931,7 @@ def validate_ignorable_clues(rule_directories): for rules_dir in rule_directories: r = list(load_rules( rules_data_dir=rules_dir, + is_builtin=is_builtin, )) combined_rules.append(r) # flatten lists of rules into a single iterable @@ -1012,6 +1028,7 @@ def build_rule_from_license(license_obj): has_stored_minimum_coverage=bool(minimum_coverage), minimum_coverage=minimum_coverage, + is_builtin=license_obj.is_builtin, is_from_license=True, is_license_text=True, @@ -1064,7 +1081,7 @@ def get_license_tokens(): yield 'licensed' -def load_rules(rules_data_dir=rules_data_dir, with_checks=True): +def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True): """ Return an iterable of rules loaded from rule files in ``rules_data_dir``. Optionally check for consistency if ``with_checks`` is True. @@ -1088,7 +1105,7 @@ def load_rules(rules_data_dir=rules_data_dir, with_checks=True): text_file = join(rules_data_dir, f'{base_name}.RULE') try: - yield Rule.from_files(data_file=data_file, text_file=text_file) + yield Rule.from_files(data_file=data_file, text_file=text_file, is_builtin=is_builtin) except Exception as re: if with_checks: model_errors.append(str(re)) @@ -1193,6 +1210,13 @@ class BasicRule: 'string.') ) + is_builtin = attr.ib( + default=True, + repr=False, + metadata=dict( + help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.') + ) + # The is_license_xxx flags below are nn indication of what this rule # importance is (e.g. how important is its text when detected as a licensing # clue) as one of several "is_license_xxx" flags. These flags are mutually @@ -1877,12 +1901,12 @@ def __attrs_post_init__(self, *args, **kwargs): self.setup() @classmethod - def from_files(cls, data_file, text_file): + def from_files(cls, data_file, text_file, is_builtin): """ Return a new Rule object loaded from a data file stored at ``data_file`` and a companion ``text_file``. """ - rule = Rule() + rule = Rule(is_builtin=is_builtin) rule.load_data(data_file=data_file, text_file=text_file) return rule diff --git a/src/scancode/api.py b/src/scancode/api.py index dc17f4e5320..66047deac98 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -249,13 +249,19 @@ def _licenses_data_from_match( result['is_exception'] = lic.is_exception result['is_unknown'] = lic.is_unknown result['owner'] = lic.owner - result['homepage_url'] = lic.homepage_url - result['text_url'] = lic.text_urls[0] if lic.text_urls else '' - result['reference_url'] = license_url_template.format(lic.key) - # TODO: change this in the case of a private license? - result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key) - result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key) - + # if the license is not builtin these should all be empty + if lic.is_builtin: + result['homepage_url'] = lic.homepage_url + result['text_url'] = lic.text_urls[0] if lic.text_urls else '' + result['reference_url'] = license_url_template.format(lic.key) + result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key) + result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key) + else: + result['homepage_url'] = '' + result['text_url'] = '' + result['reference_url'] = '' + result['scancode_text_url'] = '' + result['scancode_data_url'] = '' spdx_key = lic.spdx_license_key result['spdx_license_key'] = spdx_key diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json index f50623609a6..6203b22d4c6 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json @@ -51,9 +51,9 @@ "owner": "NexB", "homepage_url": null, "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", "spdx_license_key": "scancode-example1", "spdx_url": "https://spdx.org/licenses/scancode-example1", "start_line": 1, diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index b4fce4d33a0..15c96b9fb4c 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -52,9 +52,9 @@ "owner": "NexB", "homepage_url": null, "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", "spdx_license_key": "scancode-example1", "spdx_url": "https://spdx.org/licenses/scancode-example1", "start_line": 1, diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index 483f64fb391..2dcd28883d3 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -27,7 +27,6 @@ "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" }, "additional_directories": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" ], "spdx_license_list_version": "3.17", @@ -51,9 +50,9 @@ "owner": "NexB", "homepage_url": null, "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", "spdx_license_key": "scancode-example-installed1", "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json index 07e48d87e10..43a47618e85 100644 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json @@ -21,6 +21,9 @@ "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" }, + "additional_directories": [ + "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" + ], "spdx_license_list_version": "3.17", "files_count": 1 } @@ -42,9 +45,9 @@ "owner": "NexB", "homepage_url": "", "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", "spdx_license_key": "scancode-example-installed1", "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, From 3762ca5190f362562217d08bc4a4d137c5419204 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 4 Sep 2022 20:05:18 -0700 Subject: [PATCH 29/43] fix methods based on previous changes Signed-off-by: Kevin Ji --- src/licensedcode/models.py | 41 +++++++++---------- .../data/models/licenses.dump.expected.json | 10 +++++ .../data/models/licenses.load.expected.json | 10 +++++ .../scan_multiple.expected.json | 6 +-- .../scan.expected.json | 4 ++ tests/licensedcode/test_plugin_license.py | 2 +- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index c679973108c..ef88c416641 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -903,21 +903,22 @@ def validate_additional_license_data(additional_directories, scancode_license_di raise InvalidLicense('\n'.join(message)) -def _ignorable_clue_error(rule): +def _ignorable_clue_error(rule, rules_dir): """ Return a pair of the result of validating a rule's ignorable clues and expected ignorable clues if there is an error. Otherwise, returns None. """ - result = get_ignorables(rule.text_file) + result = get_ignorables(rule.text_file(rules_data_dir=rules_dir)) expected = get_normalized_ignorables(rule) if result != expected: - data_file = rule.data_file + data_file = rule.data_file(rules_data_dir=rules_dir) + text_file = rule.text_file(rules_data_dir=rules_dir) if not data_file: - data_file = rule.text_file.replace('.LICENSE', '.yml') + data_file = text_file.replace('.LICENSE', '.yml') result['files'] = [ f'file://{data_file}', - f'file://{rule.text_file}', + f'file://{text_file}', ] return result, expected @@ -927,27 +928,23 @@ def validate_ignorable_clues(rule_directories, is_builtin): Raises an exception if any ignorable clues declared in a Rule are improperly detected in the rule text file. """ - combined_rules = [] + messages = ['Errors while validating ignorable rules:'] + error_present = False for rules_dir in rule_directories: r = list(load_rules( rules_data_dir=rules_dir, is_builtin=is_builtin, )) - combined_rules.append(r) - # flatten lists of rules into a single iterable - rules = list(chain.from_iterable(combined_rules)) - messages = ['Errors while validating ignorable rules:'] - error_present = False - for rule in rules: - if _ignorable_clue_error(rule): - error_present = True - result, expected = _ignorable_clue_error(rule) - message.append('') - message.append(f'{rule!r}') - message.append('Result:') - message.append(result) - message.append('Expected:') - message.append(expected) + for rule in r: + if _ignorable_clue_error(rule, rules_dir): + error_present = True + result, expected = _ignorable_clue_error(rule, rules_dir) + message.append('') + message.append(f'{rule!r}') + message.append('Result:') + message.append(result) + message.append('Expected:') + message.append(expected) if error_present: raise InvalidRule('\n'.join(message)) @@ -1901,7 +1898,7 @@ def __attrs_post_init__(self, *args, **kwargs): self.setup() @classmethod - def from_files(cls, data_file, text_file, is_builtin): + def from_files(cls, data_file, text_file, is_builtin=True): """ Return a new Rule object loaded from a data file stored at ``data_file`` and a companion ``text_file``. diff --git a/tests/licensedcode/data/models/licenses.dump.expected.json b/tests/licensedcode/data/models/licenses.dump.expected.json index d40d73d1f87..2f8161da57c 100644 --- a/tests/licensedcode/data/models/licenses.dump.expected.json +++ b/tests/licensedcode/data/models/licenses.dump.expected.json @@ -6,6 +6,7 @@ "category": "Permissive", "owner": "Apache Software Foundation", "homepage_url": "http://www.apache.org/licenses/", + "is_builtin": true, "spdx_license_key": "Apache-2.0", "text_urls": [ "http://www.apache.org/licenses/LICENSE-2.0" @@ -20,6 +21,7 @@ "category": "Permissive", "owner": "Carrot2", "homepage_url": "http://www.carrot2.org/carrot2.LICENSE", + "is_builtin": true, "spdx_license_key": "LicenseRef-scancode-bsd-ack-carrot2", "minimum_coverage": 80 }, @@ -30,6 +32,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-1.0.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: February 1989.\n", "spdx_license_key": "GPL-1.0", "text_urls": [ @@ -47,6 +50,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: February 1989.\n", "spdx_license_key": "GPL-1.0+" }, @@ -57,6 +61,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "is_builtin": true, "notes": "This is the last version of the GPL text as published by the FSF. This license was released: June 1991 This license is OSI certified.\n", "spdx_license_key": "GPL-2.0", "text_urls": [ @@ -77,6 +82,7 @@ "short_name": "GPL 2.0 with Library exception", "name": "GNU General Public License 2.0 with Library exception", "category": "Copyleft Limited", + "is_builtin": true, "owner": "Grammatica", "is_exception": true, "spdx_license_key": "LicenseRef-scancode-gpl-2.0-library", @@ -91,6 +97,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: June 1991 This license is OSI certified.\n", "spdx_license_key": "GPL-2.0+" }, @@ -101,6 +108,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-3.0.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: 29 June 2007 This license is OSI certified.\n", "spdx_license_key": "GPL-3.0", "text_urls": [ @@ -120,6 +128,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-3.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: 29 June 2007 This license is OSI certified.\n", "spdx_license_key": "GPL-3.0+" }, @@ -130,6 +139,7 @@ "category": "Free Restricted", "owner": "W3C - World Wide Web Consortium", "homepage_url": "http://www.w3.org/Consortium/Legal/copyright-documents-19990405", + "is_builtin": true, "spdx_license_key": "LicenseRef-scancode-w3c-docs-19990405" } ] \ No newline at end of file diff --git a/tests/licensedcode/data/models/licenses.load.expected.json b/tests/licensedcode/data/models/licenses.load.expected.json index e242729dd6b..c0f34e2f7ec 100644 --- a/tests/licensedcode/data/models/licenses.load.expected.json +++ b/tests/licensedcode/data/models/licenses.load.expected.json @@ -6,6 +6,7 @@ "category": "Permissive", "owner": "Apache Software Foundation", "homepage_url": "http://www.apache.org/licenses/", + "is_builtin": true, "spdx_license_key": "Apache-2.0", "text_urls": [ "http://www.apache.org/licenses/LICENSE-2.0" @@ -20,6 +21,7 @@ "category": "Permissive", "owner": "Carrot2", "homepage_url": "http://www.carrot2.org/carrot2.LICENSE", + "is_builtin": true, "spdx_license_key": "LicenseRef-scancode-bsd-ack-carrot2", "minimum_coverage": 80 }, @@ -30,6 +32,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-1.0.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: February 1989.", "spdx_license_key": "GPL-1.0", "text_urls": [ @@ -47,6 +50,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: February 1989.", "spdx_license_key": "GPL-1.0+" }, @@ -57,6 +61,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "is_builtin": true, "notes": "This is the last version of the GPL text as published by the FSF. This license was released: June 1991 This license is OSI certified.\n", "spdx_license_key": "GPL-2.0", "text_urls": [ @@ -77,6 +82,7 @@ "short_name": "GPL 2.0 with Library exception", "name": "GNU General Public License 2.0 with Library exception", "category": "Copyleft Limited", + "is_builtin": true, "owner": "Grammatica", "is_exception": true, "spdx_license_key": "LicenseRef-scancode-gpl-2.0-library", @@ -91,6 +97,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: June 1991 This license is OSI certified.", "spdx_license_key": "GPL-2.0+" }, @@ -101,6 +108,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-3.0.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: 29 June 2007 This license is OSI certified.", "spdx_license_key": "GPL-3.0", "text_urls": [ @@ -120,6 +128,7 @@ "category": "Copyleft", "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/licenses/gpl-3.0-standalone.html", + "is_builtin": true, "notes": "notes from SPDX:\nThis license was released: 29 June 2007 This license is OSI certified.", "spdx_license_key": "GPL-3.0+" }, @@ -130,6 +139,7 @@ "category": "Free Restricted", "owner": "W3C - World Wide Web Consortium", "homepage_url": "http://www.w3.org/Consortium/Legal/copyright-documents-19990405", + "is_builtin": true, "spdx_license_key": "LicenseRef-scancode-w3c-docs-19990405" } ] \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json index 15c96b9fb4c..e841d39336e 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json @@ -90,9 +90,9 @@ "owner": "NexB", "homepage_url": null, "text_url": "", - "reference_url": "https://scancode-licensedb.aboutcode.org/example2", - "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example2.LICENSE", - "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example2.yml", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", "spdx_license_key": "scancode-example2", "spdx_url": "https://spdx.org/licenses/scancode-example2", "start_line": 1, diff --git a/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json b/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json index 909db5edcd2..04d63faab7c 100644 --- a/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json +++ b/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json @@ -94,6 +94,7 @@ "owner": "Apache Software Foundation", "homepage_url": "http://www.apache.org/licenses/", "notes": "Per SPDX.org, this version was released January 2004 This license is OSI\ncertified\n", + "is_builtin": "yes", "spdx_license_key": "Apache-2.0", "other_spdx_license_keys": [ "LicenseRef-Apache", @@ -120,6 +121,7 @@ "owner": "Perl Foundation", "homepage_url": "http://www.perlfoundation.org/", "notes": "Per SPDX.org, this version was released 2006 This license is OSI\ncertifified.\n", + "is_builtin": "yes", "spdx_license_key": "Artistic-2.0", "osi_license_key": "Artistic-2.0", "text_urls": [ @@ -144,6 +146,7 @@ "owner": "Regents of the University of California", "homepage_url": "http://www.opensource.org/licenses/BSD-2-Clause", "notes": "Per SPDX.org, this license is OSI certified.", + "is_builtin": "yes", "spdx_license_key": "BSD-2-Clause", "other_spdx_license_keys": [ "BSD-2-Clause-NetBSD", @@ -168,6 +171,7 @@ "owner": "MIT", "homepage_url": "http://opensource.org/licenses/mit-license.php", "notes": "Per SPDX.org, this license is OSI certified.", + "is_builtin": "yes", "spdx_license_key": "MIT", "text_urls": [ "http://opensource.org/licenses/mit-license.php" diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 6de55f27b28..a79cd77862e 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -266,7 +266,7 @@ def test_detection_with_single_external_license_directory(): check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) -@pytest.mark.scanslow +# @pytest.mark.scanslow def test_detection_with_multiple_external_license_directories(): test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) example1_dir = test_env.get_test_loc('example_external_licenses/example1') From ba9740b387f01c8bfc81426e71c3468a12cf1a05 Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 4 Sep 2022 23:07:20 -0700 Subject: [PATCH 30/43] add new license provider plugin for additional licenses Signed-off-by: Kevin Ji --- .../additional_license_location_provider.py | 161 ++++++++++++++++++ src/licensedcode/models.py | 10 +- .../licenses_to_install1/setup.py | 2 +- .../src/licenses_to_install1/__init__.py | 4 +- 4 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 src/licensedcode/additional_license_location_provider.py diff --git a/src/licensedcode/additional_license_location_provider.py b/src/licensedcode/additional_license_location_provider.py new file mode 100644 index 00000000000..778958cbdd3 --- /dev/null +++ b/src/licensedcode/additional_license_location_provider.py @@ -0,0 +1,161 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/plugincode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. + +import logging +import os +import sys + +from pluggy import PluginManager as PluggyPluginManager + +from plugincode import HookimplMarker +from plugincode import HookspecMarker + +""" +Support for plugins that provide one or more paths keys typically OS-specific +paths to bundled pre-built binaries provided as Python packages. +Plugin can either be enabled for very specific environment/platform markers (OS, +arch, etc) in their built wheels .... Or be smart about OS/ARCH/etc and provide +a path based on running some code. +""" + +logger = logging.getLogger(__name__) + +# uncomment to enable logging locally +# logging.basicConfig(stream=sys.stdout) +# logger.setLevel(logging.DEBUG) + + +def logger_debug(*args): + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + + +project_name = __name__ +entrypoint = "scancode_additional_license_location_provider" + +location_provider_spec = HookspecMarker(project_name=project_name) +location_provider_impl = HookimplMarker(project_name=project_name) + + +@location_provider_spec +class AdditionalLicenseLocationProviderPlugin(object): + """ + Base plugin class for plugins that provide path locations for one or more + keys such as the path location to a native binary executable or related + system files. + A plugin is configured as it own package with proper environemnt markers + """ + + # name string under which this plugin is registered. + # This is set automatically when a plugin class is loaded in its manager. + # Subclasses must not set this. + name = None + + def get_locations(self): + """ + Return a mapping of {key: location} where location is an absolute path + to a file or directory referenced by a known key. The location should + exist on a given platorm/OS where this plgin can be installed. + """ + raise NotImplementedError + + +class AdditionalLicensePluginManager(object): + """ + A PluginManager class for simple, non-scanning related plugins. + """ + + def __init__(self, project_name, entrypoint, plugin_base_class): + """ + Initialize this plugin manager for the fully qualified Python module + name `module_qname` with plugins loaded from the setuptools `entrypoint` + that must subclass `plugin_base_class`. + """ + self.manager = PluggyPluginManager(project_name=project_name) + self.entrypoint = entrypoint + self.plugin_base_class = plugin_base_class + self.manager.add_hookspecs(sys.modules[project_name]) + + # set to True once this manager is initialized by running its setup() + self.initialized = False + + # mapping of {plugin.name: plugin_class} for all the loaded plugins of + # this manager + self.plugin_classes = dict() + + def setup(self): + """ + Load and validate available plugins for this PluginManager from its + assigned `entrypoint`. Raise an Exception if a plugin is not valid such + that when it does not subcclass the manager `plugin_base_class`. + Must be called once to initialize the plugins if this manager. + Return a list of all plugin classes for this manager. + """ + if self.initialized: + return self.plugin_classes.values() + + entrypoint = self.entrypoint + self.manager.load_setuptools_entrypoints(entrypoint) + + plugin_classes = [] + for name, plugin_class in self.manager.list_name_plugin(): + if not issubclass(plugin_class, self.plugin_base_class): + plugin_base_class = self.plugin_base_class + raise Exception( + "Invalid plugin: %(name)r: %(plugin_class)r " + "must extend %(plugin_base_class)r." % locals() + ) + + plugin_class.name = name + plugin_classes.append(plugin_class) + + self.plugin_classes = dict([(cls.name, cls) for cls in plugin_classes]) + self.initialized = True + return self.plugin_classes.values() + + +additional_license_location_provider_plugins = AdditionalLicensePluginManager( + project_name=project_name, entrypoint=entrypoint, plugin_base_class=AdditionalLicenseLocationProviderPlugin +) + + +class ProvidedLocationError(Exception): + pass + + +def get_location(location_key, _cached_locations={}): + """ + Return the location for a `location_key` if available from plugins or None. + """ + if not _cached_locations: + additional_license_location_provider_plugins.setup() + + unknown_locations = {} + + for k, plugin_class in additional_license_location_provider_plugins.plugin_classes.items(): + pc = plugin_class() + provided_locs = pc.get_locations() or {} + for loc_key, location in provided_locs.items(): + if not os.path.exists(location): + unknown_locations[loc_key] = location + + if loc_key in _cached_locations: + existing = _cached_locations[loc_key] + msg = ( + "Duplicate location key provided: {loc_key}: " + "new: {location}, existing:{existing}" + ) + msg = msg.format(**locals()) + raise ProvidedLocationError(msg) + + _cached_locations[loc_key] = location + + if unknown_locations: + msg = "Non-existing locations provided:\n:" + msg += "\n".join("key:{}, loc: {}".format(k, l) for k, l in unknown_locations.items()) + raise ProvidedLocationError(msg) + + return _cached_locations.get(location_key) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index ef88c416641..8d57fcefea7 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -812,14 +812,12 @@ def get_paths_to_installed_licenses_and_rules(): with a common prefix. """ from importlib_metadata import entry_points - from plugincode.location_provider import get_location - installed_plugins = entry_points(group='scancode_location_provider') + from licensedcode.additional_license_location_provider import get_location + installed_plugins = entry_points(group='scancode_additional_license_location_provider') paths = [] for plugin in installed_plugins: - if plugin.name.startswith(EXTERNAL_LICENSE_PLUGIN_PREFIX): - # get path to directory of licenses and/or rules - location_key = plugin.name - paths.append(get_location(location_key)) + # get path to directory of licenses and/or rules + paths.append(get_location(plugin.name)) return paths diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py index bc713a11e73..48f5e5f65f2 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py @@ -41,7 +41,7 @@ 'scancode-toolkit', ], entry_points={ - 'scancode_location_provider': [ + 'scancode_additional_license_location_provider': [ 'licenses_to_install1 = licenses_to_install1:LicensesToInstall1Paths', ], }, diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py index 1fdba1cfb9e..24ac5028c97 100644 --- a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py +++ b/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py @@ -25,10 +25,10 @@ from os.path import abspath from os.path import dirname -from plugincode.location_provider import LocationProviderPlugin +from licensedcode.additional_license_location_provider import AdditionalLicenseLocationProviderPlugin -class LicensesToInstall1Paths(LocationProviderPlugin): +class LicensesToInstall1Paths(AdditionalLicenseLocationProviderPlugin): def get_locations(self): curr_dir = dirname(abspath(__file__)) locations = { From ba11f059f10599c8410de3a6d768cf072fe84fc9 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 30 Sep 2022 12:44:42 +0200 Subject: [PATCH 31/43] Test that additional license plugin works This test can run only in the CI and use a new pytest marker. This requires isolation as it installs new license from a new location plugin. Co-authored-by: Kevin Ji Co-authored-by: Ayan Sinha Mahapatra Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 11 +++ conftest.py | 19 +++- ...azure-posix-installed-external-license.yml | 38 -------- etc/ci/azure-posix.yml | 6 -- .../azure-win-installed-external-license.yml | 38 -------- .../plugin/licenses_to_install2/MANIFEST.in | 7 ++ .../lib/licenses_to_install2/__init__.py | 21 +++++ .../licenses/example-installed-2.LICENSE | 1 + .../licenses/example-installed-2.yml | 6 ++ .../rules/example-installed-2.RULE | 1 + .../rules/example-installed-2.yml | 2 + .../plugin/licenses_to_install2/setup.cfg | 6 ++ .../plugin/licenses_to_install2/setup.py | 45 +++++++++ .../src/licenses_to_install2/__init__.py | 21 +++++ .../licenses/example-installed-2.LICENSE | 1 + .../licenses/example-installed-2.yml | 6 ++ .../rules/example-installed-2.RULE | 1 + .../rules/example-installed-2.yml | 2 + .../plugin/licenses_to_install2/test.LICENSE | 1 + .../test_file.txt | 19 ++++ .../test_file.txt.expected.json | 94 +++++++++++++++++++ ...st_additional_license_location_provider.py | 47 ++++++++++ 22 files changed, 309 insertions(+), 84 deletions(-) delete mode 100644 etc/ci/azure-posix-installed-external-license.yml delete mode 100644 etc/ci/azure-win-installed-external-license.yml create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml create mode 100644 tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE create mode 100644 tests/licensedcode/data/additional_license_location_provider/test_file.txt create mode 100644 tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json create mode 100644 tests/licensedcode/test_additional_license_location_provider.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 72920ce23c9..d5afcf2775a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -83,6 +83,17 @@ jobs: venv/bin/pytest -vvs --test-suite=all \ tests/licensedcode/test_installed_license_detection.py + # this test runs in isolation because it modifies the actual + # license index with additional licenses provided by a plugin + # and we use the special --test-suite=plugins marker for these + # tests + additional_licenses_plugin: | + venv/bin/pip install tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2 + venv/bin/scancode --reindex-licenses + venv/bin/pytest -vvs --test-suite=plugins \ + tests/licensedcode/test_additional_license_location_provider.py + + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython diff --git a/conftest.py b/conftest.py index 211c130e1e8..f52964cf0b5 100644 --- a/conftest.py +++ b/conftest.py @@ -38,6 +38,7 @@ ################################################################################ SLOW_TEST = 'scanslow' VALIDATION_TEST = 'scanvalidate' +PLUGINS_TEST = 'scanplugins' def pytest_configure(config): @@ -53,8 +54,14 @@ def pytest_configure(config): ': Mark a ScanCode test as a validation test, super slow, long running test.', ) + config.addinivalue_line( + 'markers', + PLUGINS_TEST + + ': Mark a ScanCode test as a special CI test to tests installing additional plugins.', + ) + -TEST_SUITES = 'standard', 'all', 'validate' +TEST_SUITES = ('standard', 'all', 'validate', 'plugins',) def pytest_addoption(parser): @@ -72,9 +79,11 @@ def pytest_addoption(parser): help='Select which test suite to run: ' '"standard" runs the standard test suite designed to run reasonably fast. ' '"all" runs "standard" and "slow" (long running) tests. ' - '"validate" runs all the tests. ' + '"validate" runs all the tests, except the "plugins" tests. ' + '"plugins" runs special plugins tests. Needs extra setup, and is used only in the CI. ' 'Use the @pytest.mark.scanslow marker to mark a test as "slow" test. ' 'Use the @pytest.mark.scanvalidate marker to mark a test as a "validate" test.' + 'Use the @pytest.mark.scanplugins marker to mark a test as a "plugins" test.' ) ################################################################################ @@ -87,6 +96,7 @@ def pytest_collection_modifyitems(config, items): test_suite = config.getvalue('test_suite') run_everything = test_suite == 'validate' run_slow_test = test_suite in ('all', 'validate') + run_only_plugins = test_suite == 'plugins' tests_to_run = [] tests_to_skip = [] @@ -94,6 +104,11 @@ def pytest_collection_modifyitems(config, items): for item in items: is_validate = bool(item.get_closest_marker(VALIDATION_TEST)) is_slow = bool(item.get_closest_marker(SLOW_TEST)) + is_plugins = bool(item.get_closest_marker(PLUGINS_TEST)) + + if is_plugins and not run_only_plugins: + tests_to_skip.append(item) + continue if is_validate and not run_everything: tests_to_skip.append(item) diff --git a/etc/ci/azure-posix-installed-external-license.yml b/etc/ci/azure-posix-installed-external-license.yml deleted file mode 100644 index 4bd7c257f4e..00000000000 --- a/etc/ci/azure-posix-installed-external-license.yml +++ /dev/null @@ -1,38 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - python_architecture: x64 - install_packages: | - venv/bin/python3 -m pip install --upgrade pip setuptools wheel - venv/bin/python3 -m pip install tests/licensedcode/data/example_external_licenses/licenses_to_install1 - - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - steps: - - checkout: self - fetchDepth: 10 - - - ${{ each pyver in parameters.python_versions }}: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ pyver }}' - architecture: '${{ parameters.python_architecture }}' - displayName: '${{ pyver }} - Install Python' - - - script: | - python${{ pyver }} --version - echo "python${{ pyver }}" > PYTHON_EXECUTABLE - ./configure --clean && ./configure --dev - displayName: '${{ pyver }} - Configure' - - - script: ${{ parameters.install_packages }} - displayName: '${{ pyver }} - Install license' - - - script: venv/bin/pytest -vvs --test-suite=all tests/licensedcode/test_installed_license_detection.py - displayName: '${{ pyver }} - Run tests' \ No newline at end of file diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 2f52a370ca4..9fdc7f1522e 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -4,9 +4,6 @@ parameters: python_versions: [] test_suites: {} python_architecture: x64 - install_packages: | - venv/bin/python3 -m pip install --upgrade pip - venv/bin/python3 -m pip install tests/licensedcode/data/example_external_licenses/licenses_to_install1 jobs: - job: ${{ parameters.job_name }} @@ -38,8 +35,5 @@ jobs: ./configure --clean && ./configure --dev displayName: '${{ pyver }} - Configure' - - script: ${{ parameters.install_packages }} - displayName: '${{ pyver }} - Install license' - - script: $(test_suite) displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win-installed-external-license.yml b/etc/ci/azure-win-installed-external-license.yml deleted file mode 100644 index c8b99bf2371..00000000000 --- a/etc/ci/azure-win-installed-external-license.yml +++ /dev/null @@ -1,38 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - python_architecture: x64 - install_packages: | - venv\Scripts\python.exe -m pip install --upgrade pip setuptools wheel - venv\Scripts\python.exe -m pip install tests\licensedcode\data\example_external_licenses\licenses_to_install1 - - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - steps: - - checkout: self - fetchDepth: 10 - - - ${{ each pyver in parameters.python_versions }}: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ pyver }}' - architecture: '${{ parameters.python_architecture }}' - displayName: '${{ pyver }} - Install Python' - - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --clean && configure --dev - displayName: '${{ pyver }} - Configure' - - - script: ${{ parameters.install_packages }} - displayName: '${{ pyver }} - Install license' - - - script: venv\Scripts\pytest -vvs --test-suite=all tests\licensedcode\test_installed_license_detection.py - displayName: '${{ pyver }} - Run tests' diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in new file mode 100644 index 00000000000..b566ab1f35d --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in @@ -0,0 +1,7 @@ +graft src +graft thirdparty + +include setup.* +include *.rst +include *LICENSE* +include *NOTICE* \ No newline at end of file diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py new file mode 100644 index 00000000000..0517ee847cb --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from os.path import abspath +from os.path import dirname + +from licensedcode.additional_license_location_provider import AdditionalLicenseLocationProviderPlugin + + +class LicensesToInstall2Paths(AdditionalLicenseLocationProviderPlugin): + def get_locations(self): + curr_dir = dirname(abspath(__file__)) + locations = { + 'licenses_to_install2': curr_dir, + } + return locations diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE new file mode 100644 index 00000000000..13658412786 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE @@ -0,0 +1 @@ +This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml new file mode 100644 index 00000000000..8eb9217e9a0 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml @@ -0,0 +1,6 @@ +key: example-installed-2 +short_name: Example Installed License 2 +name: Example Installed License 2 +category: Permissive +owner: NexB +spdx_license_key: LicenseRef-scancode-example-installed2 diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE new file mode 100644 index 00000000000..3c54d093ef2 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE @@ -0,0 +1 @@ +This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml new file mode 100644 index 00000000000..fee21dca76e --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml @@ -0,0 +1,2 @@ +license_expression: example-installed-2 +is_license_text: yes diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg new file mode 100644 index 00000000000..f5483e8ca2e --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg @@ -0,0 +1,6 @@ +[metadata] +license_files = + test.LICENSE + +[aliases] +release = clean --all bdist_wheel --plat-name manylinux1_x86_64 --python-tag py3 \ No newline at end of file diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py new file mode 100644 index 00000000000..b166e32c436 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from glob import glob +from os.path import basename +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +desc = '''A ScanCode path provider plugin to provide a set of example external licenses 2 that can be installed.''' + +setup( + name='licenses_to_install2', + version='1.0', + license='apache-2.0', + description=desc, + long_description=desc, + author='nexB', + author_email='info@aboutcode.org', + url='https://github.com/nexB/scancode-plugins', + packages=find_packages('src'), + package_dir={'': 'src'}, + include_package_data=True, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Topic :: Utilities', + ], + keywords=[ + 'open source', 'scancode_licenses', + ], + install_requires=[ + 'scancode-toolkit', + ], + entry_points={ + 'scancode_additional_license_location_provider': [ + 'licenses_to_install2 = licenses_to_install2:LicensesToInstall2Paths', + ], + }, +) \ No newline at end of file diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py new file mode 100644 index 00000000000..0517ee847cb --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from os.path import abspath +from os.path import dirname + +from licensedcode.additional_license_location_provider import AdditionalLicenseLocationProviderPlugin + + +class LicensesToInstall2Paths(AdditionalLicenseLocationProviderPlugin): + def get_locations(self): + curr_dir = dirname(abspath(__file__)) + locations = { + 'licenses_to_install2': curr_dir, + } + return locations diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE new file mode 100644 index 00000000000..13658412786 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE @@ -0,0 +1 @@ +This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml new file mode 100644 index 00000000000..8eb9217e9a0 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml @@ -0,0 +1,6 @@ +key: example-installed-2 +short_name: Example Installed License 2 +name: Example Installed License 2 +category: Permissive +owner: NexB +spdx_license_key: LicenseRef-scancode-example-installed2 diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE new file mode 100644 index 00000000000..3c54d093ef2 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE @@ -0,0 +1 @@ +This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml new file mode 100644 index 00000000000..fee21dca76e --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml @@ -0,0 +1,2 @@ +license_expression: example-installed-2 +is_license_text: yes diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE new file mode 100644 index 00000000000..6c7f04de819 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE @@ -0,0 +1 @@ +This is a test license. That's all there is to it! \ No newline at end of file diff --git a/tests/licensedcode/data/additional_license_location_provider/test_file.txt b/tests/licensedcode/data/additional_license_location_provider/test_file.txt new file mode 100644 index 00000000000..8f3f9b753d3 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/test_file.txt @@ -0,0 +1,19 @@ +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum foo bar +This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit. +Lorem ipsum install packages foo bar +Lorem ipsum foo bar +Lorem ipsum install packages foo bar +Lorem ipsum foo bar +This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit. +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar +Lorem ipsum install packages foo bar + diff --git a/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json b/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json new file mode 100644 index 00000000000..5b9a8fcb784 --- /dev/null +++ b/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json @@ -0,0 +1,94 @@ +{ + "files": [ + { + "path": "test_file.txt", + "type": "file", + "licenses": [ + { + "key": "example-installed-2", + "score": 100.0, + "name": "Example Installed License 2", + "short_name": "Example Installed License 2", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "LicenseRef-scancode-example-installed2", + "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", + "start_line": 8, + "end_line": 8, + "matched_rule": { + "identifier": "example-installed-2.LICENSE", + "license_expression": "example-installed-2", + "licenses": [ + "example-installed-2" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 12, + "matched_length": 12, + "match_coverage": 100.0, + "rule_relevance": 100 + }, + "matched_text": "This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit." + }, + { + "key": "example-installed-2", + "score": 66.0, + "name": "Example Installed License 2", + "short_name": "Example Installed License 2", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "LicenseRef-scancode-example-installed2", + "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", + "start_line": 13, + "end_line": 13, + "matched_rule": { + "identifier": "example-installed-2.RULE", + "license_expression": "example-installed-2", + "licenses": [ + "example-installed-2" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 12, + "matched_length": 12, + "match_coverage": 100.0, + "rule_relevance": 66 + }, + "matched_text": "This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit." + } + ], + "license_expressions": [ + "example-installed-2", + "example-installed-2" + ], + "percentage_of_license_text": 21.05, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/test_additional_license_location_provider.py b/tests/licensedcode/test_additional_license_location_provider.py new file mode 100644 index 00000000000..e749bc7df67 --- /dev/null +++ b/tests/licensedcode/test_additional_license_location_provider.py @@ -0,0 +1,47 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +import pytest +from commoncode.testcase import FileDrivenTesting + +from scancode.cli_test_utils import check_json_scan +from scancode.cli_test_utils import run_scan_click +from scancode_config import REGEN_TEST_FIXTURES + +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + +""" +These tests need to have an extra "additional licenses" plugin install to work. +They will fail when spaned locally therefore we use a special Pytest marker +so that they run only in the CI to avoid problems. +""" + + +@pytest.mark.scanplugins +def test_additional_license_location_provider__works_with_simple_plugin(): + test_file = test_env.get_test_loc('additional_license_location_provider/test_file.txt') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--license-text', + '--strip-root', + '--verbose', + '--json-pp', result_file, + test_file, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc( + test_path='additional_license_location_provider/test_file.txt.expected.json', + must_exist=False, + ) + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + From a4ebbe07477b9822f0fc7c726a1ecf8206dde070 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 30 Sep 2022 17:53:24 +0200 Subject: [PATCH 32/43] Use new "scanplugins" pytest marker Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 10 +++------- tests/licensedcode/test_installed_license_detection.py | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d5afcf2775a..b83a2e13e12 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,8 +32,7 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven1.py \ --ignore=tests/licensedcode/test_detection_datadriven2.py \ --ignore=tests/licensedcode/test_detection_datadriven3.py \ - --ignore=tests/licensedcode/test_detection_datadriven4.py \ - --ignore=tests/licensedcode/test_installed_license_detection.py \ + --ignore=tests/licensedcode/test_detection_datadriven4.py tests/licensedcode license_datadriven1_2: | @@ -79,10 +78,6 @@ jobs: venv/bin/pytest -n 3 -vvs --test-suite=all \ tests/licensedcode/test_zzzz_cache.py - installed_licenses: | - venv/bin/pytest -vvs --test-suite=all \ - tests/licensedcode/test_installed_license_detection.py - # this test runs in isolation because it modifies the actual # license index with additional licenses provided by a plugin # and we use the special --test-suite=plugins marker for these @@ -91,7 +86,8 @@ jobs: venv/bin/pip install tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2 venv/bin/scancode --reindex-licenses venv/bin/pytest -vvs --test-suite=plugins \ - tests/licensedcode/test_additional_license_location_provider.py + tests/licensedcode/test_additional_license_location_provider.py \ + tests/licensedcode/test_installed_license_detection.py - template: etc/ci/azure-posix.yml diff --git a/tests/licensedcode/test_installed_license_detection.py b/tests/licensedcode/test_installed_license_detection.py index 10c13e14f23..b888792c049 100644 --- a/tests/licensedcode/test_installed_license_detection.py +++ b/tests/licensedcode/test_installed_license_detection.py @@ -24,7 +24,7 @@ """ -@pytest.mark.scanslow +@pytest.mark.scanplugins def test_detection_with_single_installed_external_license(): test_dir = test_env.get_test_loc('plugin_license/installed_licenses/scan', copy=True) result_file = test_env.get_temp_file('json') @@ -40,7 +40,7 @@ def test_detection_with_single_installed_external_license(): check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) -@pytest.mark.scanslow +@pytest.mark.scanplugins def test_detection_with_single_installed_external_rule(): test_dir = test_env.get_test_loc('plugin_license/installed_rules/scan', copy=True) result_file = test_env.get_temp_file('json') From 1db9437fdba688d05fe692aaf5d4cd9cd58840be Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 30 Sep 2022 17:56:31 +0200 Subject: [PATCH 33/43] Add CHANGELOG entry Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa34f91bbf3..16f3f4a360f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -46,7 +46,6 @@ License detection: matches in a larger license detecion. This remove a larger number of false positive or ambiguous license detections. - - The data structure of the JSON output has changed for licenses. We now return match details once for each matched license expression rather than once for each license in a matched expression. There is a new top-level @@ -54,7 +53,9 @@ License detection: detected license only once. This data can contain the reference license text as an option. - +- We can now detect licenses using custom license texts and license rules. + These can be provided as a one off in a directory or packaged as a plugin + for consistent reuse and deployment. v31.1.1 - 2022-09-02 From 8df0e2710fa596009ec8f4abd19a5095b3d4952d Mon Sep 17 00:00:00 2001 From: Kevin Ji Date: Sun, 2 Oct 2022 12:08:15 -0700 Subject: [PATCH 34/43] fix expected scan results after installed license CI change Signed-off-by: Kevin Ji --- .../installed_licenses/scan.expected.json | 26 ++++++++--------- .../installed_rules/scan.expected.json | 28 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json index 2dcd28883d3..215d9cc90eb 100644 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json @@ -40,10 +40,10 @@ "type": "file", "licenses": [ { - "key": "example-installed-1", - "score": 100.0, - "name": "Example Installed License 1", - "short_name": "Example Installed License 1", + "key": "example-installed-2", + "score": 91.67, + "name": "Example Installed License 2", + "short_name": "Example Installed License 2", "category": "Permissive", "is_exception": false, "is_unknown": false, @@ -53,15 +53,15 @@ "reference_url": "", "scancode_text_url": "", "scancode_data_url": "", - "spdx_license_key": "scancode-example-installed1", - "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", + "spdx_license_key": "LicenseRef-scancode-example-installed2", + "spdx_url": "***/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", "start_line": 1, "end_line": 1, "matched_rule": { - "identifier": "example-installed-1.LICENSE", - "license_expression": "example-installed-1", + "identifier": "example-installed-2.LICENSE", + "license_expression": "example-installed-2", "licenses": [ - "example-installed-1" + "example-installed-2" ], "referenced_filenames": [], "is_license_text": true, @@ -70,16 +70,16 @@ "is_license_tag": false, "is_license_intro": false, "has_unknown": false, - "matcher": "1-hash", - "rule_length": 11, + "matcher": "3-seq", + "rule_length": 12, "matched_length": 11, - "match_coverage": 100.0, + "match_coverage": 91.67, "rule_relevance": 100 } } ], "license_expressions": [ - "example-installed-1" + "example-installed-2" ], "percentage_of_license_text": 100.0, "scan_errors": [] diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json index 43a47618e85..7d2b57f71b4 100644 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json @@ -35,10 +35,10 @@ "type": "file", "licenses": [ { - "key": "example-installed-1", - "score": 61.0, - "name": "Example Installed License 1", - "short_name": "Example Installed License 1", + "key": "example-installed-2", + "score": 60.5, + "name": "Example Installed License 2", + "short_name": "Example Installed License 2", "category": "Permissive", "is_exception": false, "is_unknown": false, @@ -48,15 +48,15 @@ "reference_url": "", "scancode_text_url": "", "scancode_data_url": "", - "spdx_license_key": "scancode-example-installed1", - "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", + "spdx_license_key": "LicenseRef-scancode-example-installed2", + "spdx_url": "***/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", "start_line": 1, "end_line": 1, "matched_rule": { - "identifier": "example-installed-1.RULE", - "license_expression": "example-installed-1", + "identifier": "example-installed-2.RULE", + "license_expression": "example-installed-2", "licenses": [ - "example-installed-1" + "example-installed-2" ], "referenced_filenames": [], "is_license_text": true, @@ -65,16 +65,16 @@ "is_license_tag": false, "is_license_intro": false, "has_unknown": false, - "matcher": "1-hash", - "rule_length": 11, + "matcher": "3-seq", + "rule_length": 12, "matched_length": 11, - "match_coverage": 100.0, - "rule_relevance": 61 + "match_coverage": 91.67, + "rule_relevance": 66 } } ], "license_expressions": [ - "example-installed-1" + "example-installed-2" ], "percentage_of_license_text": 100.0, "scan_errors": [] From f53886f9db39ca752e9ebd0aeb6fbd0ed18c31a9 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 12 Oct 2022 13:26:37 +0530 Subject: [PATCH 35/43] Reorganize additional license tests * Organize tests data under one single folder. * Organize test functions under one single file. * Keep three tests, one for additional directory, one for additional plugin, and one for combined testing. * Edit the CI file accordingly. Co-authored-by: Philippe Ombredanne Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 12 +- tests/cluecode/test_plugin_filter_clues.py | 1 - .../test_file.txt | 19 -- .../test_file.txt.expected.json | 94 ---------- .../additional_dir}/licenses/example1.LICENSE | 0 .../additional_dir}/licenses/example1.yml | 0 .../additional_dir}/licenses/example2.LICENSE | 0 .../additional_dir}/licenses/example2.yml | 0 .../additional_dir}/rules/example1_1.RULE | 0 .../additional_dir}/rules/example1_1.yml | 0 .../additional_dir}/rules/example2.RULE | 0 .../additional_dir}/rules/example2.yml | 0 ...tional_license_combined_test.expected.json | 170 ++++++++++++++++++ .../additional_license_combined_test.txt | 10 ++ ...onal_license_directory_test.expected.json} | 43 +---- .../additional_license_directory_test.txt} | 0 ...ditional_license_plugin_test.expected.json | 53 ++++++ .../additional_license_plugin_test.txt} | 0 .../additional_plugin_1}/.gitignore | 0 .../additional_plugin_1}/MANIFEST.in | 0 .../additional_plugin_1}/setup.cfg | 0 .../additional_plugin_1}/setup.py | 0 .../src/licenses_to_install1/__init__.py | 0 .../licenses/example-installed-1.LICENSE} | 0 .../licenses/example-installed-1.yml | 0 .../rules/example-installed-1.RULE | 0 .../rules/example-installed-1.yml | 0 .../additional_plugin_1}/test.LICENSE | 0 .../tests/data/example-installed-1.txt | 0 .../tests/data/example-installed-1.txt.yml | 0 .../tests/test_detection_datadriven.py | 0 .../additional_plugin_2}/MANIFEST.in | 0 .../lib/licenses_to_install2/__init__.py | 0 .../licenses/example-installed-2.LICENSE | 0 .../licenses/example-installed-2.yml | 0 .../rules/example-installed-2.RULE | 0 .../rules/example-installed-2.yml | 0 .../additional_plugin_2}/setup.cfg | 0 .../additional_plugin_2}/setup.py | 0 .../src/licenses_to_install2/__init__.py | 0 .../licenses/example-installed-2.LICENSE | 0 .../licenses/example-installed-2.yml | 0 .../rules/example-installed-2.RULE | 0 .../rules/example-installed-2.yml | 0 .../additional_plugin_2}/test.LICENSE | 0 .../licenses/apache-2.0.LICENSE | 0 .../validate_licenses/licenses/apache-2.0.yml | 0 .../licenses/bsd-ack-carrot2.LICENSE | 0 .../licenses/bsd-ack-carrot2.yml | 0 .../external_licenses/scan.expected.json | 89 --------- .../installed_rules/scan/license.txt | 1 - tests/licensedcode/test_additional_license.py | 99 ++++++++++ ...st_additional_license_location_provider.py | 47 ----- tests/licensedcode/test_plugin_license.py | 61 ------- 54 files changed, 341 insertions(+), 358 deletions(-) delete mode 100644 tests/licensedcode/data/additional_license_location_provider/test_file.txt delete mode 100644 tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json rename tests/licensedcode/data/{example_external_licenses/example1 => additional_licenses/additional_dir}/licenses/example1.LICENSE (100%) rename tests/licensedcode/data/{example_external_licenses/example1 => additional_licenses/additional_dir}/licenses/example1.yml (100%) rename tests/licensedcode/data/{example_external_licenses/example2 => additional_licenses/additional_dir}/licenses/example2.LICENSE (100%) rename tests/licensedcode/data/{example_external_licenses/example2 => additional_licenses/additional_dir}/licenses/example2.yml (100%) rename tests/licensedcode/data/{example_external_licenses/example1 => additional_licenses/additional_dir}/rules/example1_1.RULE (100%) rename tests/licensedcode/data/{example_external_licenses/example1 => additional_licenses/additional_dir}/rules/example1_1.yml (100%) rename tests/licensedcode/data/{example_external_licenses/example2 => additional_licenses/additional_dir}/rules/example2.RULE (100%) rename tests/licensedcode/data/{example_external_licenses/example2 => additional_licenses/additional_dir}/rules/example2.yml (100%) create mode 100644 tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json create mode 100644 tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt rename tests/licensedcode/data/{plugin_license/external_licenses/scan_multiple.expected.json => additional_licenses/additional_license_directory_test.expected.json} (58%) rename tests/licensedcode/data/{plugin_license/external_licenses/scan/license.txt => additional_licenses/additional_license_directory_test.txt} (100%) create mode 100644 tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.LICENSE => additional_licenses/additional_license_plugin_test.txt} (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/.gitignore (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_1}/MANIFEST.in (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_1}/setup.cfg (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/setup.py (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/src/licenses_to_install1/__init__.py (100%) rename tests/licensedcode/data/{plugin_license/installed_licenses/scan/license.txt => additional_licenses/additional_plugin_1/src/licenses_to_install1/licenses/example-installed-1.LICENSE} (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/src/licenses_to_install1/licenses/example-installed-1.yml (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/src/licenses_to_install1/rules/example-installed-1.RULE (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/src/licenses_to_install1/rules/example-installed-1.yml (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_1}/test.LICENSE (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/tests/data/example-installed-1.txt (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/tests/data/example-installed-1.txt.yml (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_1}/tests/test_detection_datadriven.py (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_2}/MANIFEST.in (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/build/lib/licenses_to_install2/__init__.py (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/build/lib/licenses_to_install2/licenses/example-installed-2.yml (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/build/lib/licenses_to_install2/rules/example-installed-2.RULE (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/build/lib/licenses_to_install2/rules/example-installed-2.yml (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_2}/setup.cfg (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/setup.py (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/src/licenses_to_install2/__init__.py (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/src/licenses_to_install2/licenses/example-installed-2.LICENSE (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/src/licenses_to_install2/licenses/example-installed-2.yml (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/src/licenses_to_install2/rules/example-installed-2.RULE (100%) rename tests/licensedcode/data/{additional_license_location_provider/plugin/licenses_to_install2 => additional_licenses/additional_plugin_2}/src/licenses_to_install2/rules/example-installed-2.yml (100%) rename tests/licensedcode/data/{example_external_licenses/licenses_to_install1 => additional_licenses/additional_plugin_2}/test.LICENSE (100%) rename tests/licensedcode/data/{plugin_license => additional_licenses}/validate_licenses/licenses/apache-2.0.LICENSE (100%) rename tests/licensedcode/data/{plugin_license => additional_licenses}/validate_licenses/licenses/apache-2.0.yml (100%) rename tests/licensedcode/data/{plugin_license => additional_licenses}/validate_licenses/licenses/bsd-ack-carrot2.LICENSE (100%) rename tests/licensedcode/data/{plugin_license => additional_licenses}/validate_licenses/licenses/bsd-ack-carrot2.yml (100%) delete mode 100644 tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json delete mode 100644 tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt create mode 100644 tests/licensedcode/test_additional_license.py delete mode 100644 tests/licensedcode/test_additional_license_location_provider.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b83a2e13e12..4f76d816b06 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -82,13 +82,13 @@ jobs: # license index with additional licenses provided by a plugin # and we use the special --test-suite=plugins marker for these # tests - additional_licenses_plugin: | - venv/bin/pip install tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2 - venv/bin/scancode --reindex-licenses + additional_license_combined: | + venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_1/ + venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_2/ + venv/bin/scancode-reindex-licenses \ + --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/ venv/bin/pytest -vvs --test-suite=plugins \ - tests/licensedcode/test_additional_license_location_provider.py \ - tests/licensedcode/test_installed_license_detection.py - + tests/licensedcode/test_additional_license.py - template: etc/ci/azure-posix.yml parameters: diff --git a/tests/cluecode/test_plugin_filter_clues.py b/tests/cluecode/test_plugin_filter_clues.py index 4b1370579c5..7e882b7f185 100644 --- a/tests/cluecode/test_plugin_filter_clues.py +++ b/tests/cluecode/test_plugin_filter_clues.py @@ -60,7 +60,6 @@ def test_scan_plugin_filter_clues_does_not_filter_incorrectly(): # Regression on types tracked in https://github.com/nexB/typecode/issues/21 -@pytest.mark.xfail# def test_scan_plugin_filter_clues_for_license(): # this test fies is a copy of pcre.LICENSE that contains # several emails, authors, urls diff --git a/tests/licensedcode/data/additional_license_location_provider/test_file.txt b/tests/licensedcode/data/additional_license_location_provider/test_file.txt deleted file mode 100644 index 8f3f9b753d3..00000000000 --- a/tests/licensedcode/data/additional_license_location_provider/test_file.txt +++ /dev/null @@ -1,19 +0,0 @@ -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum foo bar -This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit. -Lorem ipsum install packages foo bar -Lorem ipsum foo bar -Lorem ipsum install packages foo bar -Lorem ipsum foo bar -This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit. -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar -Lorem ipsum install packages foo bar - diff --git a/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json b/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json deleted file mode 100644 index 5b9a8fcb784..00000000000 --- a/tests/licensedcode/data/additional_license_location_provider/test_file.txt.expected.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "files": [ - { - "path": "test_file.txt", - "type": "file", - "licenses": [ - { - "key": "example-installed-2", - "score": 100.0, - "name": "Example Installed License 2", - "short_name": "Example Installed License 2", - "category": "Permissive", - "is_exception": false, - "is_unknown": false, - "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", - "spdx_license_key": "LicenseRef-scancode-example-installed2", - "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", - "start_line": 8, - "end_line": 8, - "matched_rule": { - "identifier": "example-installed-2.LICENSE", - "license_expression": "example-installed-2", - "licenses": [ - "example-installed-2" - ], - "referenced_filenames": [], - "is_license_text": true, - "is_license_notice": false, - "is_license_reference": false, - "is_license_tag": false, - "is_license_intro": false, - "has_unknown": false, - "matcher": "2-aho", - "rule_length": 12, - "matched_length": 12, - "match_coverage": 100.0, - "rule_relevance": 100 - }, - "matched_text": "This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit." - }, - { - "key": "example-installed-2", - "score": 66.0, - "name": "Example Installed License 2", - "short_name": "Example Installed License 2", - "category": "Permissive", - "is_exception": false, - "is_unknown": false, - "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", - "spdx_license_key": "LicenseRef-scancode-example-installed2", - "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", - "start_line": 13, - "end_line": 13, - "matched_rule": { - "identifier": "example-installed-2.RULE", - "license_expression": "example-installed-2", - "licenses": [ - "example-installed-2" - ], - "referenced_filenames": [], - "is_license_text": true, - "is_license_notice": false, - "is_license_reference": false, - "is_license_tag": false, - "is_license_intro": false, - "has_unknown": false, - "matcher": "2-aho", - "rule_length": 12, - "matched_length": 12, - "match_coverage": 100.0, - "rule_relevance": 66 - }, - "matched_text": "This is a test rule EXAMPLE2 that must be installed into ScanCode Toolkit." - } - ], - "license_expressions": [ - "example-installed-2", - "example-installed-2" - ], - "percentage_of_license_text": 21.05, - "scan_errors": [] - } - ] -} \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE b/tests/licensedcode/data/additional_licenses/additional_dir/licenses/example1.LICENSE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example1/licenses/example1.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_dir/licenses/example1.LICENSE diff --git a/tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml b/tests/licensedcode/data/additional_licenses/additional_dir/licenses/example1.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example1/licenses/example1.yml rename to tests/licensedcode/data/additional_licenses/additional_dir/licenses/example1.yml diff --git a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE b/tests/licensedcode/data/additional_licenses/additional_dir/licenses/example2.LICENSE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example2/licenses/example2.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_dir/licenses/example2.LICENSE diff --git a/tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml b/tests/licensedcode/data/additional_licenses/additional_dir/licenses/example2.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example2/licenses/example2.yml rename to tests/licensedcode/data/additional_licenses/additional_dir/licenses/example2.yml diff --git a/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE b/tests/licensedcode/data/additional_licenses/additional_dir/rules/example1_1.RULE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.RULE rename to tests/licensedcode/data/additional_licenses/additional_dir/rules/example1_1.RULE diff --git a/tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml b/tests/licensedcode/data/additional_licenses/additional_dir/rules/example1_1.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example1/rules/example1_1.yml rename to tests/licensedcode/data/additional_licenses/additional_dir/rules/example1_1.yml diff --git a/tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE b/tests/licensedcode/data/additional_licenses/additional_dir/rules/example2.RULE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example2/rules/example2.RULE rename to tests/licensedcode/data/additional_licenses/additional_dir/rules/example2.RULE diff --git a/tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml b/tests/licensedcode/data/additional_licenses/additional_dir/rules/example2.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/example2/rules/example2.yml rename to tests/licensedcode/data/additional_licenses/additional_dir/rules/example2.yml diff --git a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json new file mode 100644 index 00000000000..b334284a4b4 --- /dev/null +++ b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json @@ -0,0 +1,170 @@ +{ + "files": [ + { + "path": "additional_license_combined_test.txt", + "type": "file", + "licenses": [ + { + "key": "example-installed-1", + "score": 100.0, + "name": "Example Installed License 1", + "short_name": "Example Installed License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "scancode-example-installed1", + "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example-installed-1.LICENSE", + "license_expression": "example-installed-1", + "licenses": [ + "example-installed-1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 11, + "matched_length": 11, + "match_coverage": 100.0, + "rule_relevance": 100 + } + }, + { + "key": "example-installed-2", + "score": 100.0, + "name": "Example Installed License 2", + "short_name": "Example Installed License 2", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "LicenseRef-scancode-example-installed2", + "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", + "start_line": 3, + "end_line": 3, + "matched_rule": { + "identifier": "example-installed-2.LICENSE", + "license_expression": "example-installed-2", + "licenses": [ + "example-installed-2" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 12, + "matched_length": 12, + "match_coverage": 100.0, + "rule_relevance": 100 + } + }, + { + "key": "example1", + "score": 100.0, + "name": "Example External License 1", + "short_name": "Example External License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "scancode-example1", + "spdx_url": "https://spdx.org/licenses/scancode-example1", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "example1.LICENSE", + "license_expression": "example1", + "licenses": [ + "example1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 9, + "matched_length": 9, + "match_coverage": 100.0, + "rule_relevance": 100 + } + }, + { + "key": "example2", + "score": 100.0, + "name": "Example External License 2", + "short_name": "Example External License 2", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "scancode-example2", + "spdx_url": "https://spdx.org/licenses/scancode-example2", + "start_line": 5, + "end_line": 9, + "matched_rule": { + "identifier": "example2.LICENSE", + "license_expression": "example2", + "licenses": [ + "example2" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 69, + "matched_length": 69, + "match_coverage": 100.0, + "rule_relevance": 100 + } + } + ], + "license_expressions": [ + "example-installed-1", + "example-installed-2", + "example1", + "example2" + ], + "percentage_of_license_text": 96.19, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt new file mode 100644 index 00000000000..afc3359a32c --- /dev/null +++ b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt @@ -0,0 +1,10 @@ +This is a test license that must be installed into ScanCode Toolkit. + +This is a test license EXAMPLE2 that must be installed into ScanCode Toolkit. + +The quick brown fox jumps over the lazy dog. Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud +exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit +in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +This is a test license. diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json similarity index 58% rename from tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json rename to tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json index e841d39336e..e6f07e7a5ef 100644 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json +++ b/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json @@ -1,44 +1,7 @@ { - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "31.0.0rc1", - "options": { - "input": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/external_licenses/scan" - ], - "--json-pp": "../temp.json", - "--license": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2022-08-29T004949.561867", - "end_timestamp": "2022-08-29T004951.423696", - "output_format_version": "2.0.0", - "duration": 1.861854076385498, - "message": null, - "errors": [], - "warnings": [], - "extra_data": { - "system_environment": { - "operating_system": "linux", - "cpu_architecture": "64", - "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", - "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", - "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" - }, - "additional_directories": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example2", - "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" - ], - "spdx_license_list_version": "3.17", - "files_count": 1 - } - } - ], "files": [ { - "path": "license.txt", + "path": "additional_license_directory_test.txt", "type": "file", "licenses": [ { @@ -50,7 +13,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": null, + "homepage_url": "", "text_url": "", "reference_url": "", "scancode_text_url": "", @@ -88,7 +51,7 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": null, + "homepage_url": "", "text_url": "", "reference_url": "", "scancode_text_url": "", diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt b/tests/licensedcode/data/additional_licenses/additional_license_directory_test.txt similarity index 100% rename from tests/licensedcode/data/plugin_license/external_licenses/scan/license.txt rename to tests/licensedcode/data/additional_licenses/additional_license_directory_test.txt diff --git a/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json new file mode 100644 index 00000000000..5b39c3df6b9 --- /dev/null +++ b/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json @@ -0,0 +1,53 @@ +{ + "files": [ + { + "path": "additional_license_plugin_test.txt", + "type": "file", + "licenses": [ + { + "key": "example-installed-1", + "score": 100.0, + "name": "Example Installed License 1", + "short_name": "Example Installed License 1", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "NexB", + "homepage_url": "", + "text_url": "", + "reference_url": "", + "scancode_text_url": "", + "scancode_data_url": "", + "spdx_license_key": "scancode-example-installed1", + "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", + "start_line": 1, + "end_line": 1, + "matched_rule": { + "identifier": "example-installed-1.LICENSE", + "license_expression": "example-installed-1", + "licenses": [ + "example-installed-1" + ], + "referenced_filenames": [], + "is_license_text": true, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": false, + "is_license_intro": false, + "has_unknown": false, + "matcher": "1-hash", + "rule_length": 11, + "matched_length": 11, + "match_coverage": 100.0, + "rule_relevance": 100 + } + } + ], + "license_expressions": [ + "example-installed-1" + ], + "percentage_of_license_text": 100.0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.LICENSE b/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.txt similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_license_plugin_test.txt diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore b/tests/licensedcode/data/additional_licenses/additional_plugin_1/.gitignore similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/.gitignore rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/.gitignore diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in b/tests/licensedcode/data/additional_licenses/additional_plugin_1/MANIFEST.in similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/MANIFEST.in rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/MANIFEST.in diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg b/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.cfg similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.cfg rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.cfg diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py b/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py b/tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/__init__.py similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/__init__.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/__init__.py diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt b/tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/licenses/example-installed-1.LICENSE similarity index 100% rename from tests/licensedcode/data/plugin_license/installed_licenses/scan/license.txt rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/licenses/example-installed-1.LICENSE diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/licenses/example-installed-1.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/licenses/example-installed-1.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/licenses/example-installed-1.yml diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE b/tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/rules/example-installed-1.RULE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.RULE rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/rules/example-installed-1.RULE diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/rules/example-installed-1.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/src/licenses_to_install1/rules/example-installed-1.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/src/licenses_to_install1/rules/example-installed-1.yml diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE b/tests/licensedcode/data/additional_licenses/additional_plugin_1/test.LICENSE similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/test.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/test.LICENSE diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt b/tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/data/example-installed-1.txt similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/data/example-installed-1.txt diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/data/example-installed-1.txt.yml similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/data/example-installed-1.txt.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/data/example-installed-1.txt.yml diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py b/tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/test_detection_datadriven.py similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/tests/test_detection_datadriven.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_1/tests/test_detection_datadriven.py diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in b/tests/licensedcode/data/additional_licenses/additional_plugin_2/MANIFEST.in similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/MANIFEST.in rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/MANIFEST.in diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py b/tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/__init__.py similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/__init__.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/__init__.py diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE b/tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/licenses/example-installed-2.LICENSE diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/licenses/example-installed-2.yml similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/licenses/example-installed-2.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/licenses/example-installed-2.yml diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE b/tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/rules/example-installed-2.RULE similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.RULE rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/rules/example-installed-2.RULE diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/rules/example-installed-2.yml similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/build/lib/licenses_to_install2/rules/example-installed-2.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/build/lib/licenses_to_install2/rules/example-installed-2.yml diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg b/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.cfg similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/setup.cfg rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.cfg diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py b/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/setup.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py b/tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/__init__.py similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/__init__.py rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/__init__.py diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE b/tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/licenses/example-installed-2.LICENSE similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/licenses/example-installed-2.LICENSE diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/licenses/example-installed-2.yml similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/licenses/example-installed-2.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/licenses/example-installed-2.yml diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE b/tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/rules/example-installed-2.RULE similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.RULE rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/rules/example-installed-2.RULE diff --git a/tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml b/tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/rules/example-installed-2.yml similarity index 100% rename from tests/licensedcode/data/additional_license_location_provider/plugin/licenses_to_install2/src/licenses_to_install2/rules/example-installed-2.yml rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/src/licenses_to_install2/rules/example-installed-2.yml diff --git a/tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE b/tests/licensedcode/data/additional_licenses/additional_plugin_2/test.LICENSE similarity index 100% rename from tests/licensedcode/data/example_external_licenses/licenses_to_install1/test.LICENSE rename to tests/licensedcode/data/additional_licenses/additional_plugin_2/test.LICENSE diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.LICENSE similarity index 100% rename from tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.LICENSE rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.LICENSE diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.yml similarity index 100% rename from tests/licensedcode/data/plugin_license/validate_licenses/licenses/apache-2.0.yml rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.yml diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.LICENSE similarity index 100% rename from tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.LICENSE rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.LICENSE diff --git a/tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml similarity index 100% rename from tests/licensedcode/data/plugin_license/validate_licenses/licenses/bsd-ack-carrot2.yml rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml diff --git a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json deleted file mode 100644 index 6203b22d4c6..00000000000 --- a/tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "31.0.0rc1", - "options": { - "input": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/external_licenses/scan" - ], - "--json-pp": "../temp.json", - "--license": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2022-08-29T004616.713244", - "end_timestamp": "2022-08-29T004618.624574", - "output_format_version": "2.0.0", - "duration": 1.911355972290039, - "message": null, - "errors": [], - "warnings": [], - "extra_data": { - "system_environment": { - "operating_system": "linux", - "cpu_architecture": "64", - "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", - "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", - "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" - }, - "additional_directories": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1", - "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" - ], - "spdx_license_list_version": "3.17", - "files_count": 1 - } - } - ], - "files": [ - { - "path": "license.txt", - "type": "file", - "licenses": [ - { - "key": "example1", - "score": 100.0, - "name": "Example External License 1", - "short_name": "Example External License 1", - "category": "Permissive", - "is_exception": false, - "is_unknown": false, - "owner": "NexB", - "homepage_url": null, - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", - "spdx_license_key": "scancode-example1", - "spdx_url": "https://spdx.org/licenses/scancode-example1", - "start_line": 1, - "end_line": 1, - "matched_rule": { - "identifier": "example1.LICENSE", - "license_expression": "example1", - "licenses": [ - "example1" - ], - "referenced_filenames": [], - "is_license_text": true, - "is_license_notice": false, - "is_license_reference": false, - "is_license_tag": false, - "is_license_intro": false, - "has_unknown": false, - "matcher": "2-aho", - "rule_length": 9, - "matched_length": 9, - "match_coverage": 100.0, - "rule_relevance": 100 - } - } - ], - "license_expressions": [ - "example1" - ], - "percentage_of_license_text": 10.98, - "scan_errors": [] - } - ] -} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt b/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt deleted file mode 100644 index bb701de0a88..00000000000 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan/license.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test rule that must be installed into ScanCode Toolkit. diff --git a/tests/licensedcode/test_additional_license.py b/tests/licensedcode/test_additional_license.py new file mode 100644 index 00000000000..1582bdca08b --- /dev/null +++ b/tests/licensedcode/test_additional_license.py @@ -0,0 +1,99 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +import pytest +from commoncode.testcase import FileDrivenTesting + +from scancode.cli_test_utils import check_json_scan +from scancode.cli_test_utils import run_scan_click +from scancode_config import REGEN_TEST_FIXTURES + +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + +def test_validate_license_library_returns_errors(): + from licensedcode.models import InvalidLicense + licenses_dir = test_env.get_test_loc('additional_licenses/validate_licenses') + args = [ + '--additional-license-directory', licenses_dir, + '--reindex-licenses', + ] + with pytest.raises(InvalidLicense): + run_scan_click(args) + + +""" +These tests need to have an extra "additional licenses" plugin install to work. +They will fail when spawned locally therefore we use a special Pytest marker +so that they run only in the CI to avoid problems. +""" + +# Testing an additional directory only +@pytest.mark.scanplugins +def test_detection_with_additional_license_directory(): + # Before running this test you need to reindex the licenses with + # the directory tests/licensedcode/data/additional_licenses/additional_dir/ + test_file = test_env.get_test_loc('additional_licenses/additional_license_directory_test.txt') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + test_file, + ] + run_scan_click(args) + expected_loc = test_env.get_test_loc('additional_licenses/additional_license_directory_test.expected.json') + check_json_scan(expected_loc, result_file, regen=REGEN_TEST_FIXTURES) + + +# Testing an additional plugin only +@pytest.mark.scanplugins +def test_detection_with_additional_license_plugin(): + # Before running this test you need to install the plugin at + # the directory tests/licensedcode/data/additional_licenses/additional_plugin_1/ + # and reindex the licenses. + test_file = test_env.get_test_loc('additional_licenses/additional_license_plugin_test.txt') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + test_file, + ] + run_scan_click(args) + expected_loc = test_env.get_test_loc('additional_licenses/additional_license_plugin_test.expected.json') + check_json_scan(expected_loc, result_file, regen=REGEN_TEST_FIXTURES) + + +# Testing an additional directory and two additional plugins together +@pytest.mark.scanplugins +def test_detection_with_additional_license_combined(): + # Before running this test you need to install the plugins at + # the directory tests/licensedcode/data/additional_licenses/additional_plugin_1/ + # the directory tests/licensedcode/data/additional_licenses/additional_plugin_2/ + # and reindex the licenses with + # the directory tests/licensedcode/data/additional_licenses/additional_dir/ + test_file = test_env.get_test_loc('additional_licenses/additional_license_combined_test.txt') + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--strip-root', + '--verbose', + '--json', result_file, + test_file, + ] + run_scan_click(args) + expected_loc = test_env.get_test_loc('additional_licenses/additional_license_combined_test.expected.json') + check_json_scan(expected_loc, result_file, regen=REGEN_TEST_FIXTURES) + diff --git a/tests/licensedcode/test_additional_license_location_provider.py b/tests/licensedcode/test_additional_license_location_provider.py deleted file mode 100644 index e749bc7df67..00000000000 --- a/tests/licensedcode/test_additional_license_location_provider.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os - -import pytest -from commoncode.testcase import FileDrivenTesting - -from scancode.cli_test_utils import check_json_scan -from scancode.cli_test_utils import run_scan_click -from scancode_config import REGEN_TEST_FIXTURES - -test_env = FileDrivenTesting() -test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - -""" -These tests need to have an extra "additional licenses" plugin install to work. -They will fail when spaned locally therefore we use a special Pytest marker -so that they run only in the CI to avoid problems. -""" - - -@pytest.mark.scanplugins -def test_additional_license_location_provider__works_with_simple_plugin(): - test_file = test_env.get_test_loc('additional_license_location_provider/test_file.txt') - result_file = test_env.get_temp_file('json') - args = [ - '--license', - '--license-text', - '--strip-root', - '--verbose', - '--json-pp', result_file, - test_file, - ] - run_scan_click(args) - test_loc = test_env.get_test_loc( - test_path='additional_license_location_provider/test_file.txt.expected.json', - must_exist=False, - ) - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) - diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index a79cd77862e..0d5d20a469a 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -242,56 +242,6 @@ def test_reindex_licenses_works(): run_scan_click(['--reindex-licenses']) -@pytest.mark.scanslow -def test_detection_with_single_external_license_directory(): - test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) - example1_dir = test_env.get_test_loc('example_external_licenses/example1') - result_file = test_env.get_temp_file('json') - - # first reindex with the example directory - run_scan_click([ - '--additional-license-directory', example1_dir, - '--reindex-licenses', - ]) - - args = [ - '--license', - '--strip-root', - '--verbose', - '--json', result_file, - test_dir, - ] - run_scan_click(args) - test_loc = test_env.get_test_loc('plugin_license/external_licenses/scan.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) - - -# @pytest.mark.scanslow -def test_detection_with_multiple_external_license_directories(): - test_dir = test_env.get_test_loc('plugin_license/external_licenses/scan', copy=True) - example1_dir = test_env.get_test_loc('example_external_licenses/example1') - example2_dir = test_env.get_test_loc('example_external_licenses/example2') - result_file = test_env.get_temp_file('json') - - # first reindex with the example directories - run_scan_click([ - '--additional-license-directory', example1_dir, - '--additional-license-directory', example2_dir, - '--reindex-licenses', - ]) - - args = [ - '--license', - '--strip-root', - '--verbose', - '--json', result_file, - test_dir, - ] - run_scan_click(args) - test_loc = test_env.get_test_loc('plugin_license/external_licenses/scan_multiple.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) - - @pytest.mark.scanslow def test_scan_license_with_url_template(): test_dir = test_env.get_test_loc('plugin_license/license_url', copy=True) @@ -325,14 +275,3 @@ def test_detection_is_correct_in_legacy_npm_package_json(): expected_file = test_env.get_test_loc('plugin_license/package/package.expected.json') run_scan_click(['-lp', '--json-pp', result_file, test_dir]) check_json_scan(expected_file, result_file, remove_uuid=True, remove_file_date=True, regen=REGEN_TEST_FIXTURES) - - -def test_validate_license_library_returns_errors(): - from licensedcode.models import InvalidLicense - licenses_dir = test_env.get_test_loc('plugin_license/validate_licenses') - args = [ - '--additional-license-directory', licenses_dir, - '--reindex-licenses', - ] - with pytest.raises(InvalidLicense): - run_scan_click(args) From 5361052117189489703d436a68e5f364f9a39c0d Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 12 Oct 2022 13:31:01 +0530 Subject: [PATCH 36/43] Move reindex licenses to a seperate script Delete the reindex licenses options from scancode CLI and add a new script which does the reindexing, also with additional directories optinally. Signed-off-by: Ayan Sinha Mahapatra --- setup-mini.cfg | 1 + setup.cfg | 1 + src/licensedcode/cache.py | 21 +++-- src/licensedcode/plugin_license.py | 81 ------------------- src/licensedcode/reindex.py | 43 ++++++++++ tests/licensedcode/test_additional_license.py | 7 +- tests/licensedcode/test_plugin_license.py | 5 +- tests/scancode/data/help/help.txt | 11 --- 8 files changed, 60 insertions(+), 110 deletions(-) create mode 100644 src/licensedcode/reindex.py diff --git a/setup-mini.cfg b/setup-mini.cfg index 67bb8ec4f4c..23c02316b61 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -146,6 +146,7 @@ packages = [options.entry_points] console_scripts = scancode = scancode.cli:scancode + scancode-reindex-licenses = licensedcode.reindex:reindex_licenses # These are configurations for ScanCode plugins as setuptools entry points. # Each plugin entry hast this form: diff --git a/setup.cfg b/setup.cfg index 09066b9f237..ca1056b2b16 100644 --- a/setup.cfg +++ b/setup.cfg @@ -146,6 +146,7 @@ packages = [options.entry_points] console_scripts = scancode = scancode.cli:scancode + scancode-reindex-licenses = licensedcode.reindex:reindex_licenses # These are configurations for ScanCode plugins as setuptools entry points. # Each plugin entry hast this form: diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index b45647a73ef..c6567b789dd 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -59,7 +59,7 @@ def load_or_build( timeout=LICENSE_INDEX_LOCK_TIMEOUT, licenses_data_dir=None, rules_data_dir=None, - additional_directories=None, + additional_directory=None, ): """ Load or build and save and return a LicenseCache object. @@ -114,12 +114,11 @@ def load_or_build( # Here, the cache is either stale or non-existing: we need to # rebuild all cached data (e.g. mostly the index) and cache it + additional_directories = get_paths_to_installed_licenses_and_rules() # include installed licenses - if not additional_directories: - additional_directories = get_paths_to_installed_licenses_and_rules() - else: + if additional_directory: # additional_directories is originally a tuple - additional_directories = list(additional_directories) + get_paths_to_installed_licenses_and_rules() + additional_directories.append(additional_directory) # persist additional directories as a file so that we can include it in the scancode output as extra info # only persist when we're generating a new license cache @@ -353,7 +352,7 @@ def build_unknown_spdx_symbol(licenses_db=None): return LicenseSymbolLike(licenses_db['unknown-spdx']) -def get_cache(force=False, index_all_languages=False, additional_directories=None): +def get_cache(force=False, index_all_languages=False, additional_directory=None): """ Return a LicenseCache either rebuilt, cached or loaded from disk. @@ -361,12 +360,12 @@ def get_cache(force=False, index_all_languages=False, additional_directories=Non building the license index. Otherwise, only include the English license \ texts and rules (the default) """ - populate_cache(force=force, index_all_languages=index_all_languages, additional_directories=additional_directories) + populate_cache(force=force, index_all_languages=index_all_languages, additional_directory=additional_directory) global _LICENSE_CACHE return _LICENSE_CACHE -def populate_cache(force=False, index_all_languages=False, additional_directories=None): +def populate_cache(force=False, index_all_languages=False, additional_directory=None): """ Load or build and cache a LicenseCache. Return None. """ @@ -380,7 +379,7 @@ def populate_cache(force=False, index_all_languages=False, additional_directorie index_all_languages=index_all_languages, # used for testing only timeout=LICENSE_INDEX_LOCK_TIMEOUT, - additional_directories=additional_directories, + additional_directory=additional_directory, ) @@ -402,14 +401,14 @@ def load_cache_file(cache_file): raise Exception(msg) from e -def get_index(force=False, index_all_languages=False, additional_directories=None): +def get_index(force=False, index_all_languages=False, additional_directory=None): """ Return and eventually build and cache a LicenseIndex. """ return get_cache( force=force, index_all_languages=index_all_languages, - additional_directories=additional_directories + additional_directory=additional_directory ).index diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 528276c7f44..46af018c2e9 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -11,8 +11,6 @@ from functools import partial import attr -import click -from commoncode.cliutils import MISC_GROUP from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import SCAN_OPTIONS_GROUP from commoncode.cliutils import SCAN_GROUP @@ -45,55 +43,6 @@ def logger_debug(*args): return prn(' '.join(isinstance(a, str) and a or repr(a) for a in args)) -def reindex_licenses(ctx, param, value): - """ - Rebuild and cache the license index - """ - if not value or ctx.resilient_parsing: - return - - # TODO: check for temp file configuration and use that for the cache!!! - from licensedcode.cache import get_index - import click - click.echo('Rebuilding the license index...') - get_index(force=True) - click.echo('Done.') - ctx.exit(0) - - -def reindex_licenses_additional(ctx, param, value): - """ - Rebuild and cache the license index using additional directories passed in - from command line. - """ - if not value or ctx.resilient_parsing: - return - - # TODO: check for temp file configuration and use that for the cache!!! - from licensedcode.cache import get_index - import click - click.echo('Rebuilding the license index with additional directories...') - get_index(force=True, additional_directories=value) - click.echo('Done.') - ctx.exit(0) - -def reindex_licenses_all_languages(ctx, param, value): - """ - EXPERIMENTAL: Rebuild and cache the license index including all languages - and not only English. - """ - if not value or ctx.resilient_parsing: - return - - # TODO: check for temp file configuration and use that for the cache!!! - from licensedcode.cache import get_index - import click - click.echo('Rebuilding the license index for all languages...') - get_index(force=True, index_all_languages=True) - click.echo('Done.') - ctx.exit(0) - - @scan_impl class LicenseScanner(ScanPlugin): """ @@ -155,36 +104,6 @@ class LicenseScanner(ScanPlugin): 'references such as "See license in file COPYING".', help_group=SCAN_OPTIONS_GROUP, ), - - PluggableCommandLineOption( - ('--reindex-licenses',), - is_flag=True, is_eager=True, - callback=reindex_licenses, - help='Rebuild the license index and exit.', - help_group=MISC_GROUP, - ), - - PluggableCommandLineOption( - ('--reindex-licenses-for-all-languages',), - is_flag=True, is_eager=True, - callback=reindex_licenses_all_languages, - help='[EXPERIMENTAL] Rebuild the license index including texts all ' - 'languages (and not only English) and exit.', - help_group=MISC_GROUP, - ), - - PluggableCommandLineOption( - ('--additional-license-directory',), - required_options=['reindex_licenses'], - multiple=True, - type=click.Path(exists=True, readable=True, file_okay=False, resolve_path=True, path_type=str), - metavar='DIR', - callback=reindex_licenses_additional, - help='Include this directory with additional custom licenses and license rules ' - 'in the license detection index.', - help_group=MISC_GROUP, - is_eager=True, - ), ] def is_enabled(self, license, **kwargs): # NOQA diff --git a/src/licensedcode/reindex.py b/src/licensedcode/reindex.py new file mode 100644 index 00000000000..47dcc94b01e --- /dev/null +++ b/src/licensedcode/reindex.py @@ -0,0 +1,43 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import click + + +@click.command(name='scancode-reindex-licenses') +@click.option( + '--all-languages', + is_flag=True, + help='[EXPERIMENTAL] Rebuild the license index including texts all ' + 'languages (and not only English) and exit.', +) +@click.option( + '--additional-directory', + type=click.Path(exists=True, readable=True, file_okay=False, resolve_path=True, path_type=str), + metavar='DIR', + help='Include this directory with additional custom licenses and license rules ' + 'in the license detection index.', +) +@click.help_option('-h', '--help') +def reindex_licenses( + all_languages, + additional_directory, + *args, + **kwargs, +): + """Reindex scancode licenses and exit""" + + from licensedcode.cache import get_index + click.echo('Rebuilding the license index...') + get_index(force=True, index_all_languages=bool(all_languages), additional_directory=additional_directory) + click.echo('Done.') + + +if __name__ == '__main__': + reindex_licenses() diff --git a/tests/licensedcode/test_additional_license.py b/tests/licensedcode/test_additional_license.py index 1582bdca08b..087d581c835 100644 --- a/tests/licensedcode/test_additional_license.py +++ b/tests/licensedcode/test_additional_license.py @@ -22,13 +22,10 @@ def test_validate_license_library_returns_errors(): from licensedcode.models import InvalidLicense + from licensedcode.cache import get_index licenses_dir = test_env.get_test_loc('additional_licenses/validate_licenses') - args = [ - '--additional-license-directory', licenses_dir, - '--reindex-licenses', - ] with pytest.raises(InvalidLicense): - run_scan_click(args) + get_index(force=True, additional_directory=licenses_dir) """ diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 0d5d20a469a..6d75a245375 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -238,8 +238,9 @@ def test_match_reference_license(): def test_reindex_licenses_works(): - run_scan_click(['--reindex-licenses-for-all-languages']) - run_scan_click(['--reindex-licenses']) + from licensedcode.cache import get_index + get_index(force=True) + get_index(force=True, index_all_languages=True) @pytest.mark.scanslow diff --git a/tests/scancode/data/help/help.txt b/tests/scancode/data/help/help.txt index 160e60e93bc..a51211ff899 100644 --- a/tests/scancode/data/help/help.txt +++ b/tests/scancode/data/help/help.txt @@ -157,17 +157,6 @@ Options: and including the starting directory. Use 0 for no scan depth limit. - miscellaneous: - --additional-license-directory DIR - Include this directory with additional - custom licenses and license rules in the - license detection index. - --reindex-licenses Rebuild the license index and exit. - --reindex-licenses-for-all-languages - [EXPERIMENTAL] Rebuild the license index - including texts all languages (and not only - English) and exit. - documentation: -h, --help Show this message and exit. -A, --about Show information about ScanCode and licensing and exit. From 64120395dc35a0f16e0227499e2ec855bdfd9475 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 13 Oct 2022 01:52:43 +0530 Subject: [PATCH 37/43] Add external licenses info in header Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 1 - src/licensedcode/cache.py | 54 +++--- src/licensedcode/models.py | 157 ++++++++++-------- src/licensedcode/plugin_license.py | 8 + src/scancode/cli.py | 10 -- .../licenses/bsd-ack-carrot2.yml | 1 - ...he-2.0.LICENSE => test-apache-2.0.LICENSE} | 0 .../{apache-2.0.yml => test-apache-2.0.yml} | 2 +- ...2.LICENSE => test-bsd-ack-carrot2.LICENSE} | 0 .../licenses/test-bsd-ack-carrot2.yml | 1 + .../data/cache/data/licenses/anu-license.yml | 2 +- .../cache/data/licenses/public-domain.yml | 2 +- .../data/cache/data/licenses/unknown-spdx.yml | 1 + .../data/cache/data/licenses/unknown.yml | 1 + .../test_installed_license_detection.py | 56 ------- 15 files changed, 129 insertions(+), 167 deletions(-) delete mode 100644 tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml rename tests/licensedcode/data/additional_licenses/validate_licenses/licenses/{apache-2.0.LICENSE => test-apache-2.0.LICENSE} (100%) rename tests/licensedcode/data/additional_licenses/validate_licenses/licenses/{apache-2.0.yml => test-apache-2.0.yml} (94%) rename tests/licensedcode/data/additional_licenses/validate_licenses/licenses/{bsd-ack-carrot2.LICENSE => test-bsd-ack-carrot2.LICENSE} (100%) create mode 100644 tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.yml delete mode 100644 tests/licensedcode/test_installed_license_detection.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4f76d816b06..dc200a74566 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,7 +33,6 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven2.py \ --ignore=tests/licensedcode/test_detection_datadriven3.py \ --ignore=tests/licensedcode/test_detection_datadriven4.py - tests/licensedcode license_datadriven1_2: | venv/bin/pytest -n 3 -vvs --test-suite=all \ diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index c6567b789dd..000accc8257 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -6,6 +6,7 @@ # See https://github.com/nexB/scancode-toolkit for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import click import os import pickle @@ -35,7 +36,6 @@ LICENSE_INDEX_FILENAME = 'index_cache' LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile' LICENSE_CHECKSUM_FILE = 'scancode_license_index_tree_checksums' -CACHED_DIRECTORIES_FILENAME = 'cached_directories' @attr.s(slots=True) @@ -48,6 +48,8 @@ class LicenseCache: licensing = attribute(help='Licensing object') spdx_symbols = attribute(help='mapping of LicenseSymbol objects by SPDX key') unknown_spdx_symbol = attribute(help='LicenseSymbol object') + additional_license_directory = attribute(help='Path to an additional license directory used in the license detection') + additional_license_plugins = attribute(help='Path to additional license plugins used in the license detection') @staticmethod def load_or_build( @@ -72,8 +74,11 @@ def load_or_build( directories containing licenses that are not present in the existing cache. - If the cache does not exist, a new index is built and cached. - If ``index_all_languages`` is True, include texts in all languages when - building the license index. Otherwise, only include the English license \ + building the license index. Otherwise, only include the English license texts and rules (the default) + - ``additional_directory`` is an optional additional directory + that contain additional licenses and rules in a /licenses and a /rules + directories using the same format that we use for licenses and rules. """ idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) create_dir(idx_cache_dir) @@ -114,26 +119,25 @@ def load_or_build( # Here, the cache is either stale or non-existing: we need to # rebuild all cached data (e.g. mostly the index) and cache it - additional_directories = get_paths_to_installed_licenses_and_rules() + additional_directories = [] + plugin_directories = get_paths_to_installed_licenses_and_rules() + if plugin_directories: + additional_directories.extend(plugin_directories) + # include installed licenses if additional_directory: # additional_directories is originally a tuple additional_directories.append(additional_directory) - # persist additional directories as a file so that we can include it in the scancode output as extra info - # only persist when we're generating a new license cache - idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) - cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) - with open(cached_directories_file, 'wb') as file: - pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL) - additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) - validate_additional_license_data(additional_directories=additional_license_dirs, scancode_license_dir=licenses_data_dir) - combined_directories = [licenses_data_dir] + additional_license_dirs - licenses_db = load_licenses_from_multiple_dirs( - license_directories=combined_directories, + validate_additional_license_data( + additional_directories=additional_license_dirs, scancode_license_dir=licenses_data_dir ) + licenses_db = load_licenses_from_multiple_dirs( + additional_license_data_dirs=additional_license_dirs, + builtin_license_data_dir=licenses_data_dir, + ) # create a single merged index containing license data from licenses_data_dir # and data from additional directories @@ -142,7 +146,7 @@ def load_or_build( licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir, index_all_languages=index_all_languages, - additional_directories=additional_directories, + additional_directories=plugin_directories, ) spdx_symbols = build_spdx_symbols(licenses_db=licenses_db) @@ -155,6 +159,8 @@ def load_or_build( licensing=licensing, spdx_symbols=spdx_symbols, unknown_spdx_symbol=unknown_spdx_symbol, + additional_license_directory=additional_directory, + additional_license_plugins=plugin_directories, ) # save the cache as pickle new tree checksum @@ -186,13 +192,11 @@ def build_index( from licensedcode.index import LicenseIndex from licensedcode.models import get_license_dirs from licensedcode.models import get_rule_dirs - from licensedcode.models import get_rules from licensedcode.models import get_rules_from_multiple_dirs from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import get_license_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd - from licensedcode.models import load_licenses from licensedcode.models import load_licenses_from_multiple_dirs from licensedcode.models import validate_ignorable_clues from licensedcode.legalese import common_license_words @@ -211,11 +215,10 @@ def build_index( additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories) validate_ignorable_clues(rule_directories=additional_rule_dirs, is_builtin=False) # then combine the rules in these additional directories with the rules in the original rules directory - combined_rule_directories = [rules_data_dir] + additional_rule_dirs rules = get_rules_from_multiple_dirs( licenses_db=licenses_db, - rule_directories=combined_rule_directories, - scancode_rules_dir=rules_data_dir + additional_rules_data_dirs=additional_rule_dirs, + builtin_rule_data_dir=rules_data_dir, ) legalese = common_license_words @@ -360,14 +363,16 @@ def get_cache(force=False, index_all_languages=False, additional_directory=None) building the license index. Otherwise, only include the English license \ texts and rules (the default) """ - populate_cache(force=force, index_all_languages=index_all_languages, additional_directory=additional_directory) - global _LICENSE_CACHE - return _LICENSE_CACHE + return populate_cache( + force=force, + index_all_languages=index_all_languages, + additional_directory=additional_directory, + ) def populate_cache(force=False, index_all_languages=False, additional_directory=None): """ - Load or build and cache a LicenseCache. Return None. + Return, load or build and cache a LicenseCache. """ global _LICENSE_CACHE @@ -381,6 +386,7 @@ def populate_cache(force=False, index_all_languages=False, additional_directory= timeout=LICENSE_INDEX_LOCK_TIMEOUT, additional_directory=additional_directory, ) + return _LICENSE_CACHE def load_cache_file(cache_file): diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index 835c3ef8303..f4fa7ce7e81 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -101,8 +101,6 @@ def logger_debug(*args): CATEGORIES = FOSS_CATEGORIES | OTHER_CATEGORIES -# prefix for name of all externally installed license plugins -EXTERNAL_LICENSE_PLUGIN_PREFIX = 'licenses' @attr.s(slots=True) class License: @@ -193,7 +191,7 @@ class License: default=True, repr=False, metadata=dict( - help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.') + help='Flag set to True if this is a builtin standard license.') ) # TODO: add the license key(s) this exception applies to @@ -446,8 +444,6 @@ def write(location, byte_string): if text: write(self.text_file(licenses_data_dir=licenses_data_dir), text.encode('utf-8')) - return self - def load(self, data_file, text_file): """ Populate license data from a YAML file stored in ``data_file`` and ``text_file``. @@ -694,7 +690,7 @@ def load_licenses( licenses_data_dir=licenses_data_dir, with_deprecated=False, check_dangling=True, - is_builtin = True, + is_builtin=True, ): """ Return a mapping of {key: License} loaded from license data and text files @@ -722,9 +718,16 @@ def load_licenses( key = file_base_name(data_file) try: - lic = License.from_dir(key=key, licenses_data_dir=licenses_data_dir, is_builtin=is_builtin) + lic = License.from_dir( + key=key, + licenses_data_dir=licenses_data_dir, + is_builtin=is_builtin, + ) except Exception as e: - raise Exception(f'Failed to load license: {key} from: file://{licenses_data_dir}/{key}.yml with error: {e}') from e + raise Exception( + f'Failed to load license: {key} from: ' + f'file://{licenses_data_dir}/{key}.yml with error: {e}' + ) from e if check_dangling: used_files.add(data_file) @@ -785,22 +788,17 @@ def get_rules( return chain(licenses_as_rules, rules) -def get_license_dirs( - additional_dirs, -): +def get_license_dirs(additional_dirs): """ - Returns a list of all subdirectories containing license files within the + Return a list of all subdirectories containing license files within the input list of additional directories. These directories do not have to be absolute paths. """ - # convert to absolute path in case user passes in a relative path, which messes up building rules from licenses return [f"{str(Path(path).absolute())}/licenses" for path in additional_dirs] -def get_rule_dirs( - additional_dirs, -): +def get_rule_dirs(additional_dirs): """ - Returns a list of all subdirectories containing rule files within the + Return a list of all subdirectories containing rule files within the input list of additional directories. These directories do not have to be absolute paths. """ return [f"{str(Path(path).absolute())}/rules" for path in additional_dirs] @@ -808,7 +806,7 @@ def get_rule_dirs( def get_paths_to_installed_licenses_and_rules(): """ - Returns a list of paths to externally packaged licenses or rules that the user has + Return a list of paths to externally packaged licenses or rules that the user has installed. Gets a list of all of these licenses (installed as plugins) and then gets the plugins containing licenses by checking that their names start with a common prefix. @@ -824,59 +822,67 @@ def get_paths_to_installed_licenses_and_rules(): def load_licenses_from_multiple_dirs( - license_directories, - scancode_license_dir, + builtin_license_data_dir, + additional_license_data_dirs, with_deprecated=False, ): """ - Combines a list of directories containing additional licenses into the same mapping. - These directory paths do not need to be absolute paths. - """ - combined_licenses = {} - combined_licenses_keys = set() - for license_dir in license_directories: - is_builtin = scancode_license_dir == license_dir - licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False, is_builtin=is_builtin) - # check if two dictionaries have duplicate keys - licenses_keys = set(licenses.keys()) - duplicate_keys = combined_licenses_keys.intersection(licenses_keys) + Return a mapping of {key: License} combining a list of + ``additional_license_data_dirs`` containing additional licenses with the + builtin ``builtin_license_data_dir`` licenses into the same mapping. + """ + #raise Exception(builtin_license_data_dir) + combined_licenses = load_licenses( + licenses_data_dir=builtin_license_data_dir, + with_deprecated=with_deprecated, + is_builtin=True, + ) + for license_dir in additional_license_data_dirs: + additional_licenses = load_licenses( + licenses_data_dir=license_dir, + with_deprecated=with_deprecated, + is_builtin=False, + ) + + # validate that additional licenses keys do not exist as builtin + duplicate_keys = set(combined_licenses).intersection(set(additional_licenses)) if duplicate_keys: - message = ['Duplicate licenses found when loading additional licenses.'] - message.append(', '.join(duplicate_keys)) - raise ValueError('\n'.join(message)) - combined_licenses.update(licenses) - # we keep combined_licenses_keys and licenses_keys as separate variables so we can avoid calling set() - # on the dictionary keys every single iteration - combined_licenses_keys.update(licenses_keys) + dupes = ', '.join(sorted(duplicate_keys)) + message = f'Duplicate licenses found when loading additional licenses from: {license_dir}: {dupes}' + raise ValueError(message) + + combined_licenses.update(additional_licenses) + return combined_licenses def get_rules_from_multiple_dirs( licenses_db, - rule_directories, - scancode_rules_dir, + builtin_rule_data_dir, + additional_rules_data_dirs, ): """ - Return a mapping of {key: License} built from a list of ``license_directories``. - Combines all rules together into the same data structure and validates them. - These license directories do not need to be absolute paths. - """ - if rule_directories: - combined_rules = [] - for rules_dir in rule_directories: - is_builtin = rules_dir == scancode_rules_dir - r = list(load_rules( - rules_data_dir=rules_dir, - is_builtin=is_builtin, - )) - combined_rules.append(r) - # flatten lists of rules into a single iterable - rules = list(chain.from_iterable(combined_rules)) - validate_rules(rules, licenses_db) - licenses_as_rules = build_rules_from_licenses(licenses_db) - return chain(licenses_as_rules, rules) - else: - return get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) + Yield Rule(s) built from: + - A ``license_db`` mapping of {key: License} + - The ``builtin_rule_data_dir`` of builtin license rules + - The list of ``additional_rules_data_dirs`` containing additional rules. + """ + # first load all builtin + combined_rules = list(get_rules( + licenses_db=licenses_db, + rules_data_dir=builtin_rule_data_dir, + )) + + # load additional rules + for rules_dir in additional_rules_data_dirs or []: + combined_rules.extend(load_rules( + rules_data_dir=rules_dir, + is_builtin=False, + )) + + validate_rules(rules=combined_rules, licenses_by_key=licenses_db) + + return combined_rules class InvalidLicense(Exception): @@ -888,7 +894,10 @@ def validate_additional_license_data(additional_directories, scancode_license_di Raises an exception if there are any invalid licenses in the directories of additional licenses. """ - licenses = load_licenses_from_multiple_dirs(additional_directories, scancode_license_dir) + licenses = load_licenses_from_multiple_dirs( + additional_license_data_dirs=additional_directories, + builtin_license_data_dir=scancode_license_dir + ) errors, _, _ = License.validate( licenses, verbose=False, @@ -939,14 +948,14 @@ def validate_ignorable_clues(rule_directories, is_builtin): if _ignorable_clue_error(rule, rules_dir): error_present = True result, expected = _ignorable_clue_error(rule, rules_dir) - message.append('') - message.append(f'{rule!r}') - message.append('Result:') - message.append(result) - message.append('Expected:') - message.append(expected) + messages.append('') + messages.append(f'{rule!r}') + messages.append('Result:') + messages.append(result) + messages.append('Expected:') + messages.append(expected) if error_present: - raise InvalidRule('\n'.join(message)) + raise InvalidRule('\n'.join(messages)) class InvalidRule(Exception): @@ -1102,7 +1111,11 @@ def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True) text_file = join(rules_data_dir, f'{base_name}.RULE') try: - yield Rule.from_files(data_file=data_file, text_file=text_file, is_builtin=is_builtin) + yield Rule.from_files( + data_file=data_file, + text_file=text_file, + is_builtin=is_builtin, + ) except Exception as re: if with_checks: model_errors.append(str(re)) @@ -1211,7 +1224,7 @@ class BasicRule: default=True, repr=False, metadata=dict( - help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.') + help='Flag set to True if this is a builtin standard rule.') ) # The is_license_xxx flags below are nn indication of what this rule @@ -2104,10 +2117,10 @@ def write(location, byte_string): data_file = self.data_file(rules_data_dir=rules_data_dir) as_yaml = saneyaml.dump(self.to_dict(), indent=4, encoding='utf-8') - write(location=data_file, byte_string=as_yaml) + write(data_file, as_yaml) text_file = self.text_file(rules_data_dir=rules_data_dir) - write(location=text_file, byte_string=self.text.encode('utf-8')) + write(text_file, self.text.encode('utf-8')) def load(self, data_file, text_file, with_checks=True): """ diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 46af018c2e9..f815cb41c2f 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -163,6 +163,14 @@ def process_codebase(self, codebase, unknown_licenses, **kwargs): f'after : {license_expressions_after}' ) + from licensedcode import cache + cche = cache.get_cache() + cle = codebase.get_or_create_current_header() + if cche.additional_license_directory: + cle.extra_data['additional_license_directory'] = cche.additional_license_directory + if cche.additional_license_plugins: + cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins + def add_referenced_filenames_license_matches(resource, codebase): """ diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 188af532e94..80e2bc369d7 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -862,16 +862,6 @@ def echo_func(*_args, **_kwargs): # useful for debugging cle.extra_data['system_environment'] = system_environment = {} - from licensedcode.cache import CACHED_DIRECTORIES_FILENAME, LICENSE_INDEX_DIR - import pickle - idx_cache_dir = os.path.join(scancode_config.licensedcode_cache_dir, LICENSE_INDEX_DIR) - cached_directories_file = os.path.join(idx_cache_dir, CACHED_DIRECTORIES_FILENAME) - cached_additional_directories = {} - if os.path.exists(cached_directories_file): - with open(cached_directories_file, 'rb') as file: - cached_additional_directories = pickle.load(file) - cle.extra_data['additional_directories'] = cached_additional_directories - system_environment['operating_system'] = commoncode.system.current_os system_environment['cpu_architecture'] = commoncode.system.current_arch system_environment['platform'] = platform.platform() diff --git a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml deleted file mode 100644 index 2f7cbaf8fab..00000000000 --- a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.yml +++ /dev/null @@ -1 +0,0 @@ -key: bsd-ack-carrot2 diff --git a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.LICENSE b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-apache-2.0.LICENSE similarity index 100% rename from tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.LICENSE rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-apache-2.0.LICENSE diff --git a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.yml b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-apache-2.0.yml similarity index 94% rename from tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.yml rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-apache-2.0.yml index abeb5d2fb3c..bf429e47547 100644 --- a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/apache-2.0.yml +++ b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-apache-2.0.yml @@ -1,4 +1,4 @@ -key: apache-2.0 +key: test-apache-2.0 short_name: Apache 2.0 name: Apache License 2.0 category: Permissive diff --git a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.LICENSE b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.LICENSE similarity index 100% rename from tests/licensedcode/data/additional_licenses/validate_licenses/licenses/bsd-ack-carrot2.LICENSE rename to tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.LICENSE diff --git a/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.yml b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.yml new file mode 100644 index 00000000000..55ea8536e4b --- /dev/null +++ b/tests/licensedcode/data/additional_licenses/validate_licenses/licenses/test-bsd-ack-carrot2.yml @@ -0,0 +1 @@ +key: test-bsd-ack-carrot2 diff --git a/tests/licensedcode/data/cache/data/licenses/anu-license.yml b/tests/licensedcode/data/cache/data/licenses/anu-license.yml index b882cd3fb46..6abd0c4ffff 100644 --- a/tests/licensedcode/data/cache/data/licenses/anu-license.yml +++ b/tests/licensedcode/data/cache/data/licenses/anu-license.yml @@ -1,7 +1,7 @@ key: anu-license short_name: ANU License name: Australian National University License - +spdx_license_key: LicenseRef-scancode-anu-license category: Permissive owner: ANU Data Mining Group other_urls: diff --git a/tests/licensedcode/data/cache/data/licenses/public-domain.yml b/tests/licensedcode/data/cache/data/licenses/public-domain.yml index 4c18396893c..2f7a0734165 100644 --- a/tests/licensedcode/data/cache/data/licenses/public-domain.yml +++ b/tests/licensedcode/data/cache/data/licenses/public-domain.yml @@ -1,7 +1,7 @@ key: public-domain short_name: Public Domain name: Public Domain - +spdx_license_key: LicenseRef-scancode-public-domain category: Public Domain owner: Unspecified homepage_url: http://www.linfo.org/publicdomain.html diff --git a/tests/licensedcode/data/cache/data/licenses/unknown-spdx.yml b/tests/licensedcode/data/cache/data/licenses/unknown-spdx.yml index 251792bd21f..36eea084958 100644 --- a/tests/licensedcode/data/cache/data/licenses/unknown-spdx.yml +++ b/tests/licensedcode/data/cache/data/licenses/unknown-spdx.yml @@ -3,3 +3,4 @@ short_name: unknown SPDX name: Unknown SPDX license detected, but not recognized category: Unstated License owner: Unspecified +spdx_license_key: LicenseRef-scancode-unknown-spdx \ No newline at end of file diff --git a/tests/licensedcode/data/cache/data/licenses/unknown.yml b/tests/licensedcode/data/cache/data/licenses/unknown.yml index 377d35cb668..dab5846e3fa 100644 --- a/tests/licensedcode/data/cache/data/licenses/unknown.yml +++ b/tests/licensedcode/data/cache/data/licenses/unknown.yml @@ -3,3 +3,4 @@ short_name: unknown name: Unknown license detected, but not recognized category: Unstated License owner: Unspecified +spdx_license_key: LicenseRef-scancode-unknown \ No newline at end of file diff --git a/tests/licensedcode/test_installed_license_detection.py b/tests/licensedcode/test_installed_license_detection.py deleted file mode 100644 index b888792c049..00000000000 --- a/tests/licensedcode/test_installed_license_detection.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os - -import pytest -from commoncode.testcase import FileDrivenTesting - -from scancode.cli_test_utils import check_json_scan -from scancode.cli_test_utils import run_scan_click -from scancode_config import REGEN_TEST_FIXTURES - -test_env = FileDrivenTesting() -test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - -""" -These tests spawn new process as if launched from the command line. -""" - - -@pytest.mark.scanplugins -def test_detection_with_single_installed_external_license(): - test_dir = test_env.get_test_loc('plugin_license/installed_licenses/scan', copy=True) - result_file = test_env.get_temp_file('json') - args = [ - '--license', - '--strip-root', - '--verbose', - '--json', result_file, - test_dir, - ] - run_scan_click(args) - test_loc = test_env.get_test_loc('plugin_license/installed_licenses/scan.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) - - -@pytest.mark.scanplugins -def test_detection_with_single_installed_external_rule(): - test_dir = test_env.get_test_loc('plugin_license/installed_rules/scan', copy=True) - result_file = test_env.get_temp_file('json') - args = [ - '--license', - '--strip-root', - '--verbose', - '--json', result_file, - test_dir, - ] - run_scan_click(args) - test_loc = test_env.get_test_loc('plugin_license/installed_rules/scan.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) From 6e14d8a6249fd341cea4fec26018bd7f36318c59 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 20 Oct 2022 19:20:27 +0530 Subject: [PATCH 38/43] Add is_builtin flag to matched_rule data Signed-off-by: Ayan Sinha Mahapatra --- src/licensedcode/plugin_license.py | 46 +++++++-- ...tional_license_combined_test.expected.json | 96 ++++++++++++++----- .../additional_license_combined_test.txt | 2 + ...ional_license_directory_test.expected.json | 26 ++--- ...ditional_license_plugin_test.expected.json | 13 +-- 5 files changed, 132 insertions(+), 51 deletions(-) diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index f815cb41c2f..06c8c66a121 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -143,6 +143,19 @@ def process_codebase(self, codebase, unknown_licenses, **kwargs): This is an EXPERIMENTAL feature for now. """ + from licensedcode import cache + cche = cache.get_cache() + cle = codebase.get_or_create_current_header() + licenses = cache.get_licenses_db() + has_additional_licenses = False + + if cche.additional_license_directory: + cle.extra_data['additional_license_directory'] = cche.additional_license_directory + has_additional_licenses = True + if cche.additional_license_plugins: + cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins + has_additional_licenses = True + if unknown_licenses: if codebase.has_single_resource: return @@ -154,6 +167,9 @@ def process_codebase(self, codebase, unknown_licenses, **kwargs): modified = add_referenced_filenames_license_matches(resource, codebase) + if has_additional_licenses and resource.is_file and resource.license_detections: + add_builtin_license_flag(resource, licenses) + if TRACE and modified: license_expressions_after = list(resource.license_expressions) logger_debug( @@ -163,13 +179,29 @@ def process_codebase(self, codebase, unknown_licenses, **kwargs): f'after : {license_expressions_after}' ) - from licensedcode import cache - cche = cache.get_cache() - cle = codebase.get_or_create_current_header() - if cche.additional_license_directory: - cle.extra_data['additional_license_directory'] = cche.additional_license_directory - if cche.additional_license_plugins: - cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins + +def add_builtin_license_flag(resource, licenses): + """ + Add a `is_builtin` flag to each license rule data mapping if there are + additional licenses present in the cache, either through an additional + license directory or additional license plugins. + """ + for detection in resource.license_detections: + matches = detection['matches'] + for match in matches: + add_builtin_value(license_match=match, licenses=licenses) + + for match in resource.license_clues: + add_builtin_value(license_match=match, licenses=licenses) + + +def add_builtin_value(license_match, licenses): + license_key = license_match['key'] + lic = licenses.get(license_key) + if lic.is_builtin: + license_match['is_builtin'] = True + else: + license_match['is_builtin'] = False def add_referenced_filenames_license_matches(resource, codebase): diff --git a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json index b334284a4b4..040f1b17102 100644 --- a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json +++ b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json @@ -13,11 +13,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example-installed1", "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, @@ -39,7 +39,8 @@ "rule_length": 11, "matched_length": 11, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } }, { @@ -51,11 +52,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "LicenseRef-scancode-example-installed2", "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", "start_line": 3, @@ -77,7 +78,8 @@ "rule_length": 12, "matched_length": 12, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } }, { @@ -89,11 +91,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example1", "spdx_url": "https://spdx.org/licenses/scancode-example1", "start_line": 5, @@ -115,7 +117,8 @@ "rule_length": 9, "matched_length": 9, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } }, { @@ -127,11 +130,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example2", "spdx_url": "https://spdx.org/licenses/scancode-example2", "start_line": 5, @@ -153,7 +156,47 @@ "rule_length": 69, "matched_length": 69, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false + } + }, + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "is_unknown": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://scancode-licensedb.aboutcode.org/apache-2.0", + "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/apache-2.0.LICENSE", + "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/apache-2.0.yml", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 12, + "end_line": 12, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "referenced_filenames": [], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "is_license_intro": false, + "has_unknown": false, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100, + "is_builtin": true } } ], @@ -161,9 +204,10 @@ "example-installed-1", "example-installed-2", "example1", - "example2" + "example2", + "apache-2.0" ], - "percentage_of_license_text": 96.19, + "percentage_of_license_text": 96.33, "scan_errors": [] } ] diff --git a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt index afc3359a32c..35e4a2e0b66 100644 --- a/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt +++ b/tests/licensedcode/data/additional_licenses/additional_license_combined_test.txt @@ -8,3 +8,5 @@ exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. This is a test license. + +License: Apache-2.0 diff --git a/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json index e6f07e7a5ef..2ae9fcfcdf1 100644 --- a/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json +++ b/tests/licensedcode/data/additional_licenses/additional_license_directory_test.expected.json @@ -13,11 +13,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example1", "spdx_url": "https://spdx.org/licenses/scancode-example1", "start_line": 1, @@ -39,7 +39,8 @@ "rule_length": 9, "matched_length": 9, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } }, { @@ -51,11 +52,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example2", "spdx_url": "https://spdx.org/licenses/scancode-example2", "start_line": 1, @@ -77,7 +78,8 @@ "rule_length": 69, "matched_length": 69, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } } ], diff --git a/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json b/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json index 5b39c3df6b9..1f5dbb0524f 100644 --- a/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json +++ b/tests/licensedcode/data/additional_licenses/additional_license_plugin_test.expected.json @@ -13,11 +13,11 @@ "is_exception": false, "is_unknown": false, "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", + "homepage_url": null, + "text_url": null, + "reference_url": null, + "scancode_text_url": null, + "scancode_data_url": null, "spdx_license_key": "scancode-example-installed1", "spdx_url": "https://spdx.org/licenses/scancode-example-installed1", "start_line": 1, @@ -39,7 +39,8 @@ "rule_length": 11, "matched_length": 11, "match_coverage": 100.0, - "rule_relevance": 100 + "rule_relevance": 100, + "is_builtin": false } } ], From 044f60d4b642081d2721f0253bdf7748db8ac2ca Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 20 Oct 2022 19:20:59 +0530 Subject: [PATCH 39/43] Do not return empty strings in license data Signed-off-by: Ayan Sinha Mahapatra --- docs/source/cli-reference/output-format.rst | 2 +- docs/source/cli-reference/synopsis.rst | 2 +- src/licensedcode/plugin_license.py | 80 ++-- src/scancode/api.py | 15 +- .../data/csv/flatten_scan/full.json | 32 +- .../data/csv/flatten_scan/full.json-expected | 32 +- .../data/csv/flatten_scan/minimal.json | 8 +- .../csv/flatten_scan/minimal.json-expected | 8 +- .../data/csv/non-standard/identified.json | 2 +- tests/formattedcode/data/reuse/vb.json | 24 +- .../installed_licenses/scan.expected.json | 88 ----- .../installed_rules/scan.expected.json | 83 ----- .../license-expression/scan.expected.json | 2 +- .../license-ref-see-copying.expected.json | 2 +- .../license_reference/scan-ref.expected.json | 2 +- .../license_reference/scan-wref.expected.json | 2 +- .../sqlite/sqlite.expected.json | 272 +++++++------- .../text/scan-diag.expected.json | 4 +- .../plugin_license/text/scan.expected.json | 4 +- .../text_long_lines/scan-diag.expected.json | 2 +- .../text_long_lines/scan.expected.json | 2 +- .../plugin_license_text/scan.expected.json | 6 +- .../scan.expected.json | 36 +- .../component-package-build-expected.json | 4 +- .../component-package-expected.json | 4 +- .../license-holder-rollup-expected.json | 4 +- ...return-nested-local-majority-expected.json | 12 +- .../data/virtual_idempotent/codebase.json | 24 +- .../data/full_summary/summary.expected.json | 2 +- .../summary_by_facet.expected.json | 2 +- .../summary_details.expected.json | 2 +- .../e2fsprogs-expected.json | 342 +++++++++--------- .../data/plugin_consolidate/e2fsprogs.json | 342 +++++++++--------- .../license-holder-rollup-expected.json | 2 +- ...return-nested-local-majority-expected.json | 6 +- .../plugin_consolidate/zlib-expected.json | 70 ++-- .../data/plugin_consolidate/zlib.json | 70 ++-- .../full_tallies/tallies.expected.json | 2 +- .../tallies_by_facet.expected.json | 2 +- .../tallies_details.expected.json | 2 +- ...lies_key_files-details.expected.json-lines | 2 +- .../tallies_key_files.expected.json | 2 +- 42 files changed, 702 insertions(+), 904 deletions(-) delete mode 100644 tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json delete mode 100644 tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json diff --git a/docs/source/cli-reference/output-format.rst b/docs/source/cli-reference/output-format.rst index 6941ee3a97f..f52b192c7d8 100644 --- a/docs/source/cli-reference/output-format.rst +++ b/docs/source/cli-reference/output-format.rst @@ -183,7 +183,7 @@ following options. "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style", "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { diff --git a/docs/source/cli-reference/synopsis.rst b/docs/source/cli-reference/synopsis.rst index 0d029ebbbbf..2c8d6d7ff61 100644 --- a/docs/source/cli-reference/synopsis.rst +++ b/docs/source/cli-reference/synopsis.rst @@ -234,7 +234,7 @@ A sample JSON output for an individual file will look like:: "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style", "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { diff --git a/src/licensedcode/plugin_license.py b/src/licensedcode/plugin_license.py index 06c8c66a121..cffc987a9f5 100644 --- a/src/licensedcode/plugin_license.py +++ b/src/licensedcode/plugin_license.py @@ -7,6 +7,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import logging +import os import posixpath from functools import partial @@ -20,28 +22,22 @@ from scancode.api import SCANCODE_LICENSEDB_URL -TRACE = False +TRACE = os.environ.get('SCANCODE_DEBUG_LICENSE_PLUGIN', False) -def logger_debug(*args): pass +def logger_debug(*args): + pass +logger = logging.getLogger(__name__) + if TRACE: - use_print = True - if use_print: - prn = print - else: - import logging - import sys - logger = logging.getLogger(__name__) - # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) - logging.basicConfig(stream=sys.stdout) - logger.setLevel(logging.DEBUG) - prn = logger.debug + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) def logger_debug(*args): - return prn(' '.join(isinstance(a, str) and a or repr(a) for a in args)) - + return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) @scan_impl class LicenseScanner(ScanPlugin): @@ -156,28 +152,37 @@ def process_codebase(self, codebase, unknown_licenses, **kwargs): cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins has_additional_licenses = True - if unknown_licenses: - if codebase.has_single_resource: - return + if TRACE and has_additional_licenses: + logger_debug( + f'add_referenced_filenames_license_matches: additional_licenses', + f'has_additional_licenses: {has_additional_licenses}\n', + f'additional_license_directory: {cche.additional_license_directory}\n', + f'additional_license_plugins : {cche.additional_license_plugins}' + ) - for resource in codebase.walk(topdown=False): - # follow license references to other files - if TRACE: - license_expressions_before = list(resource.license_expressions) + if codebase.has_single_resource and not codebase.root.is_file: + return + modified = False + for resource in codebase.walk(topdown=False): + # follow license references to other files + if TRACE: + license_expressions_before = list(resource.license_expressions) + + if unknown_licenses: modified = add_referenced_filenames_license_matches(resource, codebase) - if has_additional_licenses and resource.is_file and resource.license_detections: - add_builtin_license_flag(resource, licenses) + if has_additional_licenses and resource.is_file and resource.licenses: + add_builtin_license_flag(resource, licenses) - if TRACE and modified: - license_expressions_after = list(resource.license_expressions) - logger_debug( - f'add_referenced_filenames_license_matches: Modfied:', - f'{resource.path} with license_expressions:\n' - f'before: {license_expressions_before}\n' - f'after : {license_expressions_after}' - ) + if TRACE and modified: + license_expressions_after = list(resource.license_expressions) + logger_debug( + f'add_referenced_filenames_license_matches: Modfied:', + f'{resource.path} with license_expressions:\n' + f'before: {license_expressions_before}\n' + f'after : {license_expressions_after}' + ) def add_builtin_license_flag(resource, licenses): @@ -186,12 +191,7 @@ def add_builtin_license_flag(resource, licenses): additional licenses present in the cache, either through an additional license directory or additional license plugins. """ - for detection in resource.license_detections: - matches = detection['matches'] - for match in matches: - add_builtin_value(license_match=match, licenses=licenses) - - for match in resource.license_clues: + for match in resource.licenses: add_builtin_value(license_match=match, licenses=licenses) @@ -199,9 +199,9 @@ def add_builtin_value(license_match, licenses): license_key = license_match['key'] lic = licenses.get(license_key) if lic.is_builtin: - license_match['is_builtin'] = True + license_match['matched_rule']['is_builtin'] = True else: - license_match['is_builtin'] = False + license_match['matched_rule']['is_builtin'] = False def add_referenced_filenames_license_matches(resource, codebase): diff --git a/src/scancode/api.py b/src/scancode/api.py index 66047deac98..04f7e4c049e 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -249,19 +249,17 @@ def _licenses_data_from_match( result['is_exception'] = lic.is_exception result['is_unknown'] = lic.is_unknown result['owner'] = lic.owner + result['homepage_url'] = lic.homepage_url + result['text_url'] = lic.text_urls[0] if lic.text_urls else None # if the license is not builtin these should all be empty if lic.is_builtin: - result['homepage_url'] = lic.homepage_url - result['text_url'] = lic.text_urls[0] if lic.text_urls else '' result['reference_url'] = license_url_template.format(lic.key) result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key) result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key) else: - result['homepage_url'] = '' - result['text_url'] = '' - result['reference_url'] = '' - result['scancode_text_url'] = '' - result['scancode_data_url'] = '' + result['reference_url'] = None + result['scancode_text_url'] = None + result['scancode_data_url'] = None spdx_key = lic.spdx_license_key result['spdx_license_key'] = spdx_key @@ -273,7 +271,7 @@ def _licenses_data_from_match( spdx_key = lic.spdx_license_key.rstrip('+') spdx_url = SPDX_LICENSE_URL.format(spdx_key) else: - spdx_url = '' + spdx_url = None result['spdx_url'] = spdx_url result['start_line'] = match.start_line result['end_line'] = match.end_line @@ -293,7 +291,6 @@ def _licenses_data_from_match( matched_rule['matched_length'] = match.len() matched_rule['match_coverage'] = match.coverage() matched_rule['rule_relevance'] = match.rule.relevance - # FIXME: for sanity this should always be included????? if include_text: result['matched_text'] = matched_text return detected_licenses diff --git a/tests/formattedcode/data/csv/flatten_scan/full.json b/tests/formattedcode/data/csv/flatten_scan/full.json index 778da14a7c8..e95267786fb 100644 --- a/tests/formattedcode/data/csv/flatten_scan/full.json +++ b/tests/formattedcode/data/csv/flatten_scan/full.json @@ -258,11 +258,11 @@ "short_name": "JBoss EULA", "category": "Proprietary Free", "owner": "JBoss Community", - "homepage_url": "", + "homepage_url": null, "text_url": "http://repository.jboss.org/licenses/jbossorg-eula.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/jboss-eula", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 108, "matched_rule": { @@ -1296,10 +1296,10 @@ "category": "Public Domain", "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1649, "end_line": 1649, "matched_rule": { @@ -1322,10 +1322,10 @@ "category": "Public Domain", "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1692, "end_line": 1692, "matched_rule": { @@ -1478,10 +1478,10 @@ "category": "Permissive", "owner": "OpenSSL", "homepage_url": "http://openssl.org/source/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule": { @@ -1555,10 +1555,10 @@ "category": "Permissive", "owner": "OpenSSL", "homepage_url": "http://openssl.org/source/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule": { @@ -2562,11 +2562,11 @@ "short_name": "Ada linking exception to GPL 2.0 or later", "category": "Copyleft Limited", "owner": "Dmitriy Anisimkov", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 20, "end_line": 25, "matched_rule": { @@ -3268,11 +3268,11 @@ "short_name": "CMR License", "category": "Permissive", "owner": "CMR - Christian Michelsen Research AS", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/cmr-no", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { diff --git a/tests/formattedcode/data/csv/flatten_scan/full.json-expected b/tests/formattedcode/data/csv/flatten_scan/full.json-expected index 4af3c497547..e29837e4a0b 100644 --- a/tests/formattedcode/data/csv/flatten_scan/full.json-expected +++ b/tests/formattedcode/data/csv/flatten_scan/full.json-expected @@ -206,11 +206,11 @@ "license__short_name": "JBoss EULA", "license__category": "Proprietary Free", "license__owner": "JBoss Community", - "license__homepage_url": "", + "license__homepage_url": null, "license__text_url": "http://repository.jboss.org/licenses/jbossorg-eula.txt", "license__reference_url": "https://scancode-licensedb.aboutcode.org/jboss-eula", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 3, "end_line": 108, "matched_rule__identifier": "jboss-eula.LICENSE", @@ -1090,10 +1090,10 @@ "license__category": "Public Domain", "license__owner": "Unspecified", "license__homepage_url": "http://www.linfo.org/publicdomain.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 1649, "end_line": 1649, "matched_rule__identifier": "public-domain.LICENSE", @@ -1113,10 +1113,10 @@ "license__category": "Public Domain", "license__owner": "Unspecified", "license__homepage_url": "http://www.linfo.org/publicdomain.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 1692, "end_line": 1692, "matched_rule__identifier": "public-domain.LICENSE", @@ -1244,10 +1244,10 @@ "license__category": "Permissive", "license__owner": "OpenSSL", "license__homepage_url": "http://openssl.org/source/license.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule__identifier": "openssl_8.RULE", @@ -1309,10 +1309,10 @@ "license__category": "Permissive", "license__owner": "OpenSSL", "license__homepage_url": "http://openssl.org/source/license.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule__identifier": "openssl_8.RULE", @@ -2131,11 +2131,11 @@ "license__short_name": "Ada linking exception to GPL 2.0 or later", "license__category": "Copyleft Limited", "license__owner": "Dmitriy Anisimkov", - "license__homepage_url": "", - "license__text_url": "", + "license__homepage_url": null, + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 20, "end_line": 25, "matched_rule__identifier": "ada-linking-exception.LICENSE", @@ -2718,11 +2718,11 @@ "license__short_name": "CMR License", "license__category": "Permissive", "license__owner": "CMR - Christian Michelsen Research AS", - "license__homepage_url": "", - "license__text_url": "", + "license__homepage_url": null, + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/cmr-no", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule__identifier": "cmr-no.LICENSE", diff --git a/tests/formattedcode/data/csv/flatten_scan/minimal.json b/tests/formattedcode/data/csv/flatten_scan/minimal.json index 1344c78a88a..f4c59c16fa3 100644 --- a/tests/formattedcode/data/csv/flatten_scan/minimal.json +++ b/tests/formattedcode/data/csv/flatten_scan/minimal.json @@ -33,10 +33,10 @@ "category": "Permissive", "owner": "OpenSSL", "homepage_url": "http://openssl.org/source/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule": { @@ -78,10 +78,10 @@ "category": "Permissive", "owner": "OpenSSL", "homepage_url": "http://openssl.org/source/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule": { diff --git a/tests/formattedcode/data/csv/flatten_scan/minimal.json-expected b/tests/formattedcode/data/csv/flatten_scan/minimal.json-expected index 8e764283078..414a34f3ab7 100644 --- a/tests/formattedcode/data/csv/flatten_scan/minimal.json-expected +++ b/tests/formattedcode/data/csv/flatten_scan/minimal.json-expected @@ -22,10 +22,10 @@ "license__category": "Permissive", "license__owner": "OpenSSL", "license__homepage_url": "http://openssl.org/source/license.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule__identifier": "openssl_8.RULE", @@ -57,10 +57,10 @@ "license__category": "Permissive", "license__owner": "OpenSSL", "license__homepage_url": "http://openssl.org/source/license.html", - "license__text_url": "", + "license__text_url": null, "license__reference_url": "https://scancode-licensedb.aboutcode.org/openssl", "license__spdx_license_key": "", - "license__spdx_url": "", + "license__spdx_url": null, "start_line": 4, "end_line": 7, "matched_rule__identifier": "openssl_8.RULE", diff --git a/tests/formattedcode/data/csv/non-standard/identified.json b/tests/formattedcode/data/csv/non-standard/identified.json index 3dd9c531e03..6ddbed00dc9 100644 --- a/tests/formattedcode/data/csv/non-standard/identified.json +++ b/tests/formattedcode/data/csv/non-standard/identified.json @@ -69,7 +69,7 @@ "license_choices_expression": null, "license_choices": [], "reference_notes": "", - "homepage_url": "", + "homepage_url": null, "notice_text": "", "components": [ { diff --git a/tests/formattedcode/data/reuse/vb.json b/tests/formattedcode/data/reuse/vb.json index 46a656dfbf4..64f4efd2f6b 100644 --- a/tests/formattedcode/data/reuse/vb.json +++ b/tests/formattedcode/data/reuse/vb.json @@ -541,11 +541,11 @@ "is_exception": false, "is_unknown": false, "owner": "JBoss Community", - "homepage_url": "", + "homepage_url": null, "text_url": "http://repository.jboss.org/licenses/jbossorg-eula.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/jboss-eula", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 108, "matched_rule": { @@ -1557,10 +1557,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1649, "end_line": 1649, "matched_rule": { @@ -1585,10 +1585,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1692, "end_line": 1692, "matched_rule": { @@ -2270,11 +2270,11 @@ "is_exception": true, "is_unknown": false, "owner": "Dmitriy Anisimkov", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 25, "matched_rule": { @@ -2986,11 +2986,11 @@ "is_exception": false, "is_unknown": false, "owner": "CMR - Christian Michelsen Research AS", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/cmr-no", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { diff --git a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json deleted file mode 100644 index 215d9cc90eb..00000000000 --- a/tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "31.0.0rc1", - "options": { - "input": [ - "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/plugin_license/installed_licenses/scan" - ], - "--json-pp": "../temp.json", - "--license": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2022-08-29T004350.528873", - "end_timestamp": "2022-08-29T004352.269691", - "output_format_version": "2.0.0", - "duration": 1.7408447265625, - "message": null, - "errors": [], - "warnings": [], - "extra_data": { - "system_environment": { - "operating_system": "linux", - "cpu_architecture": "64", - "platform": "Linux-5.15.0-46-generic-x86_64-with-glibc2.29", - "platform_version": "#49~20.04.1-Ubuntu SMP Thu Aug 4 19:15:44 UTC 2022", - "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]" - }, - "additional_directories": [ - "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" - ], - "spdx_license_list_version": "3.17", - "files_count": 1 - } - } - ], - "files": [ - { - "path": "license.txt", - "type": "file", - "licenses": [ - { - "key": "example-installed-2", - "score": 91.67, - "name": "Example Installed License 2", - "short_name": "Example Installed License 2", - "category": "Permissive", - "is_exception": false, - "is_unknown": false, - "owner": "NexB", - "homepage_url": null, - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", - "spdx_license_key": "LicenseRef-scancode-example-installed2", - "spdx_url": "***/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", - "start_line": 1, - "end_line": 1, - "matched_rule": { - "identifier": "example-installed-2.LICENSE", - "license_expression": "example-installed-2", - "licenses": [ - "example-installed-2" - ], - "referenced_filenames": [], - "is_license_text": true, - "is_license_notice": false, - "is_license_reference": false, - "is_license_tag": false, - "is_license_intro": false, - "has_unknown": false, - "matcher": "3-seq", - "rule_length": 12, - "matched_length": 11, - "match_coverage": 91.67, - "rule_relevance": 100 - } - } - ], - "license_expressions": [ - "example-installed-2" - ], - "percentage_of_license_text": 100.0, - "scan_errors": [] - } - ] -} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json b/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json deleted file mode 100644 index 7d2b57f71b4..00000000000 --- a/tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "options": { - "input": "", - "--json": "", - "--license": true, - "--strip-root": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "output_format_version": "2.0.0", - "message": null, - "errors": [], - "warnings": [], - "extra_data": { - "system_environment": { - "operating_system": "linux", - "cpu_architecture": "64", - "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", - "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", - "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" - }, - "additional_directories": [ - "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1" - ], - "spdx_license_list_version": "3.17", - "files_count": 1 - } - } - ], - "files": [ - { - "path": "license.txt", - "type": "file", - "licenses": [ - { - "key": "example-installed-2", - "score": 60.5, - "name": "Example Installed License 2", - "short_name": "Example Installed License 2", - "category": "Permissive", - "is_exception": false, - "is_unknown": false, - "owner": "NexB", - "homepage_url": "", - "text_url": "", - "reference_url": "", - "scancode_text_url": "", - "scancode_data_url": "", - "spdx_license_key": "LicenseRef-scancode-example-installed2", - "spdx_url": "***/tree/develop/src/licensedcode/data/licenses/example-installed-2.LICENSE", - "start_line": 1, - "end_line": 1, - "matched_rule": { - "identifier": "example-installed-2.RULE", - "license_expression": "example-installed-2", - "licenses": [ - "example-installed-2" - ], - "referenced_filenames": [], - "is_license_text": true, - "is_license_notice": false, - "is_license_reference": false, - "is_license_tag": false, - "is_license_intro": false, - "has_unknown": false, - "matcher": "3-seq", - "rule_length": 12, - "matched_length": 11, - "match_coverage": 91.67, - "rule_relevance": 66 - } - } - ], - "license_expressions": [ - "example-installed-2" - ], - "percentage_of_license_text": 100.0, - "scan_errors": [] - } - ] -} diff --git a/tests/licensedcode/data/plugin_license/license-expression/scan.expected.json b/tests/licensedcode/data/plugin_license/license-expression/scan.expected.json index 3e7194b16b5..6e6e57ae752 100644 --- a/tests/licensedcode/data/plugin_license/license-expression/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/license-expression/scan.expected.json @@ -103,7 +103,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", diff --git a/tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json b/tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json index 45491ebfd68..4f559a976a3 100644 --- a/tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json +++ b/tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json @@ -64,7 +64,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", diff --git a/tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json b/tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json index 584192a1970..89de003f331 100644 --- a/tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json +++ b/tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json @@ -64,7 +64,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", diff --git a/tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json b/tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json index 518e35174ff..673f1c3e267 100644 --- a/tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json +++ b/tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json @@ -14,7 +14,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", diff --git a/tests/licensedcode/data/plugin_license/sqlite/sqlite.expected.json b/tests/licensedcode/data/plugin_license/sqlite/sqlite.expected.json index 4677e3319d1..40a86e8d737 100644 --- a/tests/licensedcode/data/plugin_license/sqlite/sqlite.expected.json +++ b/tests/licensedcode/data/plugin_license/sqlite/sqlite.expected.json @@ -22,7 +22,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -61,7 +61,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -100,7 +100,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -139,7 +139,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -178,7 +178,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -217,7 +217,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -256,7 +256,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -295,7 +295,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -334,7 +334,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -373,7 +373,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -412,7 +412,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -451,7 +451,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -490,7 +490,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -529,7 +529,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -568,7 +568,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -607,7 +607,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -646,7 +646,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -685,7 +685,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -724,7 +724,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -763,7 +763,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -802,7 +802,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -841,7 +841,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -880,7 +880,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -919,7 +919,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -958,7 +958,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -997,7 +997,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1036,7 +1036,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1075,7 +1075,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1114,7 +1114,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1153,7 +1153,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1192,7 +1192,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1231,7 +1231,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1270,7 +1270,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1309,7 +1309,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1348,7 +1348,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1387,7 +1387,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1426,7 +1426,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1465,7 +1465,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1504,7 +1504,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1543,7 +1543,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1582,7 +1582,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1621,7 +1621,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1660,7 +1660,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1699,7 +1699,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1738,7 +1738,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1777,7 +1777,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1816,7 +1816,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1855,7 +1855,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1894,7 +1894,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1933,7 +1933,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -1972,7 +1972,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2011,7 +2011,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2050,7 +2050,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2089,7 +2089,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2128,7 +2128,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2167,7 +2167,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2206,7 +2206,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2245,7 +2245,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2284,7 +2284,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2323,7 +2323,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2362,7 +2362,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2401,7 +2401,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2440,7 +2440,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2479,7 +2479,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2518,7 +2518,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2557,7 +2557,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2596,7 +2596,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2635,7 +2635,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2674,7 +2674,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2713,7 +2713,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2752,7 +2752,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2791,7 +2791,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2830,7 +2830,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2869,7 +2869,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2908,7 +2908,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2947,7 +2947,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -2986,7 +2986,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3025,7 +3025,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3064,7 +3064,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3103,7 +3103,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3142,7 +3142,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3181,7 +3181,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3220,7 +3220,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3259,7 +3259,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3298,7 +3298,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3337,7 +3337,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3376,7 +3376,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3415,7 +3415,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3454,7 +3454,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3493,7 +3493,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3532,7 +3532,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3571,7 +3571,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3610,7 +3610,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3649,7 +3649,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3688,7 +3688,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3727,7 +3727,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3766,7 +3766,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3805,7 +3805,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3844,7 +3844,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3883,7 +3883,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3922,7 +3922,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -3961,7 +3961,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4000,7 +4000,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4039,7 +4039,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4078,7 +4078,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4117,7 +4117,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4156,7 +4156,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4195,7 +4195,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4234,7 +4234,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4273,7 +4273,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4312,7 +4312,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4351,7 +4351,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4390,7 +4390,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4429,7 +4429,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4468,7 +4468,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4507,7 +4507,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4546,7 +4546,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4585,7 +4585,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4624,7 +4624,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4663,7 +4663,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4702,7 +4702,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4741,7 +4741,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4780,7 +4780,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4819,7 +4819,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4858,7 +4858,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4897,7 +4897,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4936,7 +4936,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -4975,7 +4975,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5014,7 +5014,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5053,7 +5053,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5092,7 +5092,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5131,7 +5131,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5170,7 +5170,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5209,7 +5209,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5248,7 +5248,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", @@ -5287,7 +5287,7 @@ "is_unknown": false, "owner": "SQLite", "homepage_url": "https://sqlite.org/src/artifact/df5091916dbb40e6", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/blessing", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/blessing.yml", diff --git a/tests/licensedcode/data/plugin_license/text/scan-diag.expected.json b/tests/licensedcode/data/plugin_license/text/scan-diag.expected.json index fe6ec0220d8..01d2e50b188 100644 --- a/tests/licensedcode/data/plugin_license/text/scan-diag.expected.json +++ b/tests/licensedcode/data/plugin_license/text/scan-diag.expected.json @@ -55,7 +55,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", @@ -148,7 +148,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/fsf-ap.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/fsf-ap.yml", diff --git a/tests/licensedcode/data/plugin_license/text/scan.expected.json b/tests/licensedcode/data/plugin_license/text/scan.expected.json index bc0d7454980..64905fc8e66 100644 --- a/tests/licensedcode/data/plugin_license/text/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/text/scan.expected.json @@ -55,7 +55,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", @@ -148,7 +148,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/fsf-ap.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/fsf-ap.yml", diff --git a/tests/licensedcode/data/plugin_license/text_long_lines/scan-diag.expected.json b/tests/licensedcode/data/plugin_license/text_long_lines/scan-diag.expected.json index baab28de655..212137b10dc 100644 --- a/tests/licensedcode/data/plugin_license/text_long_lines/scan-diag.expected.json +++ b/tests/licensedcode/data/plugin_license/text_long_lines/scan-diag.expected.json @@ -55,7 +55,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", diff --git a/tests/licensedcode/data/plugin_license/text_long_lines/scan.expected.json b/tests/licensedcode/data/plugin_license/text_long_lines/scan.expected.json index baab28de655..212137b10dc 100644 --- a/tests/licensedcode/data/plugin_license/text_long_lines/scan.expected.json +++ b/tests/licensedcode/data/plugin_license/text_long_lines/scan.expected.json @@ -55,7 +55,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", diff --git a/tests/licensedcode/data/plugin_license_text/scan.expected.json b/tests/licensedcode/data/plugin_license_text/scan.expected.json index 2c0c890494d..3bf1a7663b3 100644 --- a/tests/licensedcode/data/plugin_license_text/scan.expected.json +++ b/tests/licensedcode/data/plugin_license_text/scan.expected.json @@ -198,7 +198,7 @@ "is_unknown": false, "owner": "JA-SIG Collaborative", "homepage_url": "http://web.archive.org/web/20040402030132/http://uportal.org/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ja-sig", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ja-sig.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ja-sig.yml", @@ -351,7 +351,7 @@ "is_unknown": false, "owner": "Linux Foundation", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/linux-syscall-exception-gpl", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/linux-syscall-exception-gpl.yml", @@ -464,7 +464,7 @@ "is_unknown": false, "owner": "JA-SIG Collaborative", "homepage_url": "http://web.archive.org/web/20040402030132/http://uportal.org/license.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ja-sig", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ja-sig.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ja-sig.yml", diff --git a/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json b/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json index 04d63faab7c..6aee6965e7b 100644 --- a/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json +++ b/tests/licensedcode/data/plugin_licenses_reference/scan.expected.json @@ -1,32 +1,4 @@ { - "headers": [ - { - "tool_name": "scancode-toolkit", - "options": { - "input": "", - "--json-pp": "", - "--license": true, - "--licenses-reference": true, - "--package": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "output_format_version": "2.0.0", - "message": null, - "errors": [], - "warnings": [], - "extra_data": { - "system_environment": { - "operating_system": "linux", - "cpu_architecture": "64", - "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic", - "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022", - "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]" - }, - "spdx_license_list_version": "3.16", - "files_count": 2 - } - } - ], "dependencies": [], "packages": [ { @@ -94,7 +66,7 @@ "owner": "Apache Software Foundation", "homepage_url": "http://www.apache.org/licenses/", "notes": "Per SPDX.org, this version was released January 2004 This license is OSI\ncertified\n", - "is_builtin": "yes", + "is_builtin": true, "spdx_license_key": "Apache-2.0", "other_spdx_license_keys": [ "LicenseRef-Apache", @@ -121,7 +93,7 @@ "owner": "Perl Foundation", "homepage_url": "http://www.perlfoundation.org/", "notes": "Per SPDX.org, this version was released 2006 This license is OSI\ncertifified.\n", - "is_builtin": "yes", + "is_builtin": true, "spdx_license_key": "Artistic-2.0", "osi_license_key": "Artistic-2.0", "text_urls": [ @@ -146,7 +118,7 @@ "owner": "Regents of the University of California", "homepage_url": "http://www.opensource.org/licenses/BSD-2-Clause", "notes": "Per SPDX.org, this license is OSI certified.", - "is_builtin": "yes", + "is_builtin": true, "spdx_license_key": "BSD-2-Clause", "other_spdx_license_keys": [ "BSD-2-Clause-NetBSD", @@ -171,7 +143,7 @@ "owner": "MIT", "homepage_url": "http://opensource.org/licenses/mit-license.php", "notes": "Per SPDX.org, this license is OSI certified.", - "is_builtin": "yes", + "is_builtin": true, "spdx_license_key": "MIT", "text_urls": [ "http://opensource.org/licenses/mit-license.php" diff --git a/tests/scancode/data/plugin_consolidate/component-package-build-expected.json b/tests/scancode/data/plugin_consolidate/component-package-build-expected.json index 0d0daa39d31..864786d2ae3 100644 --- a/tests/scancode/data/plugin_consolidate/component-package-build-expected.json +++ b/tests/scancode/data/plugin_consolidate/component-package-build-expected.json @@ -850,10 +850,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/scancode/data/plugin_consolidate/component-package-expected.json b/tests/scancode/data/plugin_consolidate/component-package-expected.json index 4cd294b034e..93a46b0253a 100644 --- a/tests/scancode/data/plugin_consolidate/component-package-expected.json +++ b/tests/scancode/data/plugin_consolidate/component-package-expected.json @@ -740,10 +740,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/scancode/data/plugin_consolidate/license-holder-rollup-expected.json b/tests/scancode/data/plugin_consolidate/license-holder-rollup-expected.json index ee5b3700ddf..33c53d59207 100644 --- a/tests/scancode/data/plugin_consolidate/license-holder-rollup-expected.json +++ b/tests/scancode/data/plugin_consolidate/license-holder-rollup-expected.json @@ -524,10 +524,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/scancode/data/plugin_consolidate/return-nested-local-majority-expected.json b/tests/scancode/data/plugin_consolidate/return-nested-local-majority-expected.json index 8823f671c65..22f75ddac53 100644 --- a/tests/scancode/data/plugin_consolidate/return-nested-local-majority-expected.json +++ b/tests/scancode/data/plugin_consolidate/return-nested-local-majority-expected.json @@ -150,10 +150,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -265,10 +265,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -528,10 +528,10 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/scancode/data/virtual_idempotent/codebase.json b/tests/scancode/data/virtual_idempotent/codebase.json index 6476625b895..babf4e1022e 100644 --- a/tests/scancode/data/virtual_idempotent/codebase.json +++ b/tests/scancode/data/virtual_idempotent/codebase.json @@ -1111,11 +1111,11 @@ "is_exception": false, "is_unknown": false, "owner": "JBoss Community", - "homepage_url": "", + "homepage_url": null, "text_url": "http://repository.jboss.org/licenses/jbossorg-eula.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/jboss-eula", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 108, "matched_rule": { @@ -2990,10 +2990,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1649, "end_line": 1649, "matched_rule": { @@ -3024,10 +3024,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 1692, "end_line": 1692, "matched_rule": { @@ -4254,11 +4254,11 @@ "is_exception": true, "is_unknown": false, "owner": "Dmitriy Anisimkov", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 25, "matched_rule": { @@ -5547,11 +5547,11 @@ "is_exception": false, "is_unknown": false, "owner": "CMR - Christian Michelsen Research AS", - "homepage_url": "", - "text_url": "", + "homepage_url": null, + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/cmr-no", "spdx_license_key": "", - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { diff --git a/tests/summarycode/data/full_summary/summary.expected.json b/tests/summarycode/data/full_summary/summary.expected.json index 2fbc42215f1..4806e9d58e2 100644 --- a/tests/summarycode/data/full_summary/summary.expected.json +++ b/tests/summarycode/data/full_summary/summary.expected.json @@ -7480,7 +7480,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/full_summary/summary_by_facet.expected.json b/tests/summarycode/data/full_summary/summary_by_facet.expected.json index d20b89202f0..ceb583da539 100644 --- a/tests/summarycode/data/full_summary/summary_by_facet.expected.json +++ b/tests/summarycode/data/full_summary/summary_by_facet.expected.json @@ -8166,7 +8166,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/full_summary/summary_details.expected.json b/tests/summarycode/data/full_summary/summary_details.expected.json index 689edd3803f..bea0719fe35 100644 --- a/tests/summarycode/data/full_summary/summary_details.expected.json +++ b/tests/summarycode/data/full_summary/summary_details.expected.json @@ -8399,7 +8399,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/plugin_consolidate/e2fsprogs-expected.json b/tests/summarycode/data/plugin_consolidate/e2fsprogs-expected.json index 77491ea9510..33eddbf4f1f 100644 --- a/tests/summarycode/data/plugin_consolidate/e2fsprogs-expected.json +++ b/tests/summarycode/data/plugin_consolidate/e2fsprogs-expected.json @@ -1716,7 +1716,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -2002,7 +2002,7 @@ "text_url": "https://developercertificate.org/", "reference_url": "https://scancode-licensedb.aboutcode.org/dco-1.1", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 8, "matched_rule": { @@ -2035,7 +2035,7 @@ "text_url": "https://developercertificate.org/", "reference_url": "https://scancode-licensedb.aboutcode.org/dco-1.1", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 10, "end_line": 34, "matched_rule": { @@ -2135,10 +2135,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://www.gnu.org/software/autoconf-archive/", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-macro-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 18, "matched_rule": { @@ -2169,7 +2169,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "spdx_license_key": "FSFAP", "spdx_url": "https://spdx.org/licenses/FSFAP", @@ -2202,7 +2202,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "spdx_license_key": "FSFAP", "spdx_url": "https://spdx.org/licenses/FSFAP", @@ -2328,7 +2328,7 @@ "text_url": "http://www.fsf.org/licensing/licenses/", "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited-no-warranty", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 12, "matched_rule": { @@ -2358,7 +2358,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2391,7 +2391,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2424,7 +2424,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2457,7 +2457,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2490,7 +2490,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2523,7 +2523,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2556,7 +2556,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2589,7 +2589,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2622,7 +2622,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2655,7 +2655,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2688,7 +2688,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2721,7 +2721,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2754,7 +2754,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2787,7 +2787,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2888,7 +2888,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2921,7 +2921,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2954,7 +2954,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2987,7 +2987,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3020,7 +3020,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3053,7 +3053,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3120,10 +3120,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception-2.0", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2865, "end_line": 2882, "matched_rule": { @@ -3154,7 +3154,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3187,7 +3187,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3220,7 +3220,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3253,7 +3253,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3286,7 +3286,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3319,7 +3319,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3352,7 +3352,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3385,7 +3385,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3418,7 +3418,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3451,7 +3451,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3484,7 +3484,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -4074,10 +4074,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 25, "matched_rule": { @@ -4108,10 +4108,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 55, "end_line": 56, "matched_rule": { @@ -4264,7 +4264,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -4392,10 +4392,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 25, "matched_rule": { @@ -4426,10 +4426,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 72, "end_line": 73, "matched_rule": { @@ -4602,7 +4602,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/libtool-exception-2.0", "spdx_license_key": "Libtool-exception", "spdx_url": "https://spdx.org/licenses/Libtool-exception", @@ -4670,7 +4670,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/libtool-exception-2.0", "spdx_license_key": "Libtool-exception", "spdx_url": "https://spdx.org/licenses/Libtool-exception", @@ -4866,10 +4866,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -6270,10 +6270,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/patent-disclaimer", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 30, "matched_rule": { @@ -7903,7 +7903,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -7939,7 +7939,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -9405,7 +9405,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -9697,7 +9697,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -15020,10 +15020,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/proprietary-license", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 44, "end_line": 44, "matched_rule": { @@ -15804,7 +15804,7 @@ "is_unknown": false, "owner": "LaTeX", "homepage_url": "https://fedoraproject.org/wiki/Licensing/Latex2e", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/latex2e", "spdx_license_key": "Latex2e", "spdx_url": "https://spdx.org/licenses/Latex2e", @@ -15837,7 +15837,7 @@ "is_unknown": false, "owner": "LaTeX", "homepage_url": "https://fedoraproject.org/wiki/Licensing/Latex2e", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/latex2e", "spdx_license_key": "Latex2e", "spdx_url": "https://spdx.org/licenses/Latex2e", @@ -15964,10 +15964,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 12, "end_line": 29, "matched_rule": { @@ -22264,10 +22264,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 4, "matched_rule": { @@ -25367,10 +25367,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/bison-exception-2.0", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 306, "end_line": 324, "matched_rule": { @@ -28107,10 +28107,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 66, "end_line": 72, "matched_rule": { @@ -33148,7 +33148,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33298,7 +33298,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33331,10 +33331,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 54, "end_line": 54, "matched_rule": { @@ -33364,7 +33364,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33633,7 +33633,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33713,7 +33713,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33857,7 +33857,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -33931,7 +33931,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -34005,7 +34005,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -34552,10 +34552,10 @@ "is_unknown": false, "owner": "Paul Mackerras", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/paul-mackerras-binary", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 40, "matched_rule": { @@ -34814,10 +34814,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 12, "end_line": 29, "matched_rule": { @@ -34964,10 +34964,10 @@ "is_unknown": false, "owner": "Regents of the University of California", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/bsla", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 15, "matched_rule": { @@ -37178,10 +37178,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 13, "end_line": 15, "matched_rule": { @@ -43408,10 +43408,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -43565,10 +43565,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -44358,10 +44358,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -44522,10 +44522,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -46467,10 +46467,10 @@ "is_unknown": false, "owner": "Theodore Ts'o", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/tso-license", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 21, "matched_rule": { @@ -46723,7 +46723,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46803,7 +46803,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46877,7 +46877,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46957,7 +46957,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47031,7 +47031,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47105,7 +47105,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47179,7 +47179,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47253,7 +47253,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47327,7 +47327,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47485,7 +47485,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47559,7 +47559,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47633,7 +47633,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47707,7 +47707,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47781,7 +47781,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47855,7 +47855,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -47999,7 +47999,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -48201,7 +48201,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -48648,10 +48648,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 15, "matched_rule": { @@ -48736,10 +48736,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 15, "matched_rule": { @@ -49204,10 +49204,10 @@ "is_unknown": false, "owner": "MIT", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-with-modification-obligations", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 47, "matched_rule": { @@ -49336,10 +49336,10 @@ "is_unknown": false, "owner": "MIT", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-with-modification-obligations", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 15, "end_line": 36, "matched_rule": { @@ -57400,10 +57400,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 9, "matched_rule": { @@ -57719,10 +57719,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -57887,10 +57887,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58055,10 +58055,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -58218,10 +58218,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58337,10 +58337,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -58565,10 +58565,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58723,10 +58723,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58885,10 +58885,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -59043,10 +59043,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -59217,10 +59217,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -59371,10 +59371,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 7, "matched_rule": { @@ -59555,10 +59555,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -59717,10 +59717,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -59875,10 +59875,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -60043,10 +60043,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -60201,10 +60201,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -60423,10 +60423,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -60572,10 +60572,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -60729,10 +60729,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -60897,10 +60897,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 4, "matched_rule": { @@ -61070,10 +61070,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -61254,10 +61254,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/summarycode/data/plugin_consolidate/e2fsprogs.json b/tests/summarycode/data/plugin_consolidate/e2fsprogs.json index 9a18f8a2952..7113aa705a8 100644 --- a/tests/summarycode/data/plugin_consolidate/e2fsprogs.json +++ b/tests/summarycode/data/plugin_consolidate/e2fsprogs.json @@ -278,10 +278,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://www.gnu.org/software/autoconf-archive/", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-macro-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 18, "matched_rule": { @@ -312,7 +312,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "spdx_license_key": "FSFAP", "spdx_url": "https://spdx.org/licenses/FSFAP", @@ -345,7 +345,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-ap", "spdx_license_key": "FSFAP", "spdx_url": "https://spdx.org/licenses/FSFAP", @@ -468,7 +468,7 @@ "text_url": "http://www.fsf.org/licensing/licenses/", "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited-no-warranty", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 12, "matched_rule": { @@ -498,7 +498,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -531,7 +531,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -564,7 +564,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -597,7 +597,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -630,7 +630,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -663,7 +663,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -696,7 +696,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -729,7 +729,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -762,7 +762,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -795,7 +795,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -828,7 +828,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -861,7 +861,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -894,7 +894,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -927,7 +927,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1028,7 +1028,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1061,7 +1061,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1094,7 +1094,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1127,7 +1127,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1160,7 +1160,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1193,7 +1193,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1260,10 +1260,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception-2.0", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2865, "end_line": 2882, "matched_rule": { @@ -1294,7 +1294,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1327,7 +1327,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1360,7 +1360,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1393,7 +1393,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1426,7 +1426,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1459,7 +1459,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1492,7 +1492,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1525,7 +1525,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1558,7 +1558,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1591,7 +1591,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -1624,7 +1624,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -2982,7 +2982,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -3265,7 +3265,7 @@ "text_url": "https://developercertificate.org/", "reference_url": "https://scancode-licensedb.aboutcode.org/dco-1.1", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 8, "matched_rule": { @@ -3298,7 +3298,7 @@ "text_url": "https://developercertificate.org/", "reference_url": "https://scancode-licensedb.aboutcode.org/dco-1.1", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 10, "end_line": 34, "matched_rule": { @@ -3542,10 +3542,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 25, "matched_rule": { @@ -3576,10 +3576,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 55, "end_line": 56, "matched_rule": { @@ -3730,7 +3730,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/fsf-unlimited", "spdx_license_key": "FSFULLR", "spdx_url": "https://spdx.org/licenses/FSFULLR", @@ -3856,10 +3856,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob;f=config.guess;h=a7448442748cc6f98a066d2d1051fad3b043761a;hb=HEAD", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/autoconf-simple-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 25, "matched_rule": { @@ -3890,10 +3890,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 72, "end_line": 73, "matched_rule": { @@ -4064,7 +4064,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/libtool-exception-2.0", "spdx_license_key": "Libtool-exception", "spdx_url": "https://spdx.org/licenses/Libtool-exception", @@ -4132,7 +4132,7 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/libtool-exception-2.0", "spdx_license_key": "Libtool-exception", "spdx_url": "https://spdx.org/licenses/Libtool-exception", @@ -4326,10 +4326,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -4924,10 +4924,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/patent-disclaimer", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 30, "matched_rule": { @@ -6764,7 +6764,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -6800,7 +6800,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -8244,7 +8244,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -8534,7 +8534,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -12379,7 +12379,7 @@ "is_unknown": false, "owner": "LaTeX", "homepage_url": "https://fedoraproject.org/wiki/Licensing/Latex2e", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/latex2e", "spdx_license_key": "Latex2e", "spdx_url": "https://spdx.org/licenses/Latex2e", @@ -12412,7 +12412,7 @@ "is_unknown": false, "owner": "LaTeX", "homepage_url": "https://fedoraproject.org/wiki/Licensing/Latex2e", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/latex2e", "spdx_license_key": "Latex2e", "spdx_url": "https://spdx.org/licenses/Latex2e", @@ -12569,10 +12569,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 12, "end_line": 29, "matched_rule": { @@ -14457,10 +14457,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/proprietary-license", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 44, "end_line": 44, "matched_rule": { @@ -21000,10 +21000,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 4, "matched_rule": { @@ -24003,10 +24003,10 @@ "is_unknown": false, "owner": "Free Software Foundation (FSF)", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/bison-exception-2.0", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 306, "end_line": 324, "matched_rule": { @@ -25652,10 +25652,10 @@ "is_unknown": false, "owner": "Theodore Ts'o", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/tso-license", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 21, "matched_rule": { @@ -26792,10 +26792,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 66, "end_line": 72, "matched_rule": { @@ -31723,7 +31723,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -31873,7 +31873,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -31906,10 +31906,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 54, "end_line": 54, "matched_rule": { @@ -31939,7 +31939,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32203,7 +32203,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32283,7 +32283,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32427,7 +32427,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32501,7 +32501,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32575,7 +32575,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -32709,10 +32709,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 12, "end_line": 29, "matched_rule": { @@ -32857,10 +32857,10 @@ "is_unknown": false, "owner": "Regents of the University of California", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/bsla", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 15, "matched_rule": { @@ -33415,10 +33415,10 @@ "is_unknown": false, "owner": "Paul Mackerras", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/paul-mackerras-binary", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 40, "matched_rule": { @@ -35667,10 +35667,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 13, "end_line": 15, "matched_rule": { @@ -41795,10 +41795,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -41950,10 +41950,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -44561,10 +44561,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -44725,10 +44725,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-copyleft", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -44933,7 +44933,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45013,7 +45013,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45087,7 +45087,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45167,7 +45167,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45241,7 +45241,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45315,7 +45315,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45389,7 +45389,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45463,7 +45463,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45569,7 +45569,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45725,7 +45725,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45799,7 +45799,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45873,7 +45873,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -45947,7 +45947,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46021,7 +46021,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46095,7 +46095,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46239,7 +46239,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46441,7 +46441,7 @@ "is_unknown": false, "owner": "MIT", "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style_.28no_advertising_without_permission.29", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style-no-advert", "spdx_license_key": "NTP", "spdx_url": "https://spdx.org/licenses/NTP", @@ -46847,10 +46847,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 15, "matched_rule": { @@ -46933,10 +46933,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 7, "end_line": 15, "matched_rule": { @@ -47426,10 +47426,10 @@ "is_unknown": false, "owner": "MIT", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-with-modification-obligations", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 47, "matched_rule": { @@ -47556,10 +47556,10 @@ "is_unknown": false, "owner": "MIT", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/mit-with-modification-obligations", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 15, "end_line": 36, "matched_rule": { @@ -55590,10 +55590,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -55756,10 +55756,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -55922,10 +55922,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -56083,10 +56083,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -56200,10 +56200,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -56426,10 +56426,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -56582,10 +56582,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -56742,10 +56742,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -56898,10 +56898,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -57069,10 +57069,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -57223,10 +57223,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 7, "matched_rule": { @@ -57405,10 +57405,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -57549,10 +57549,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 9, "matched_rule": { @@ -57721,10 +57721,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -57877,10 +57877,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58043,10 +58043,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58231,10 +58231,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -58494,10 +58494,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -58643,10 +58643,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58798,10 +58798,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 3, "end_line": 3, "matched_rule": { @@ -58964,10 +58964,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 4, "end_line": 4, "matched_rule": { @@ -59135,10 +59135,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -59316,10 +59316,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/free-unknown", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/summarycode/data/plugin_consolidate/license-holder-rollup-expected.json b/tests/summarycode/data/plugin_consolidate/license-holder-rollup-expected.json index 35c4c630048..8c16a27d50b 100644 --- a/tests/summarycode/data/plugin_consolidate/license-holder-rollup-expected.json +++ b/tests/summarycode/data/plugin_consolidate/license-holder-rollup-expected.json @@ -541,7 +541,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", diff --git a/tests/summarycode/data/plugin_consolidate/return-nested-local-majority-expected.json b/tests/summarycode/data/plugin_consolidate/return-nested-local-majority-expected.json index b6d30b4526c..1e95a666f84 100644 --- a/tests/summarycode/data/plugin_consolidate/return-nested-local-majority-expected.json +++ b/tests/summarycode/data/plugin_consolidate/return-nested-local-majority-expected.json @@ -135,7 +135,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", @@ -262,7 +262,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", @@ -551,7 +551,7 @@ "is_unknown": true, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml", diff --git a/tests/summarycode/data/plugin_consolidate/zlib-expected.json b/tests/summarycode/data/plugin_consolidate/zlib-expected.json index f8d54ab8ee1..fed73ca514d 100644 --- a/tests/summarycode/data/plugin_consolidate/zlib-expected.json +++ b/tests/summarycode/data/plugin_consolidate/zlib-expected.json @@ -1674,10 +1674,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -1803,10 +1803,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -1920,10 +1920,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2102,10 +2102,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2219,10 +2219,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2336,10 +2336,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2453,10 +2453,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2570,10 +2570,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2687,10 +2687,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -2804,10 +2804,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 25, "matched_rule": { @@ -5927,7 +5927,7 @@ "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style", "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { @@ -6076,10 +6076,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 27, "matched_rule": { @@ -8227,7 +8227,7 @@ "text_url": "ftp://ftp.info-zip.org/pub/infozip/license.html", "reference_url": "https://scancode-licensedb.aboutcode.org/info-zip-2009-01", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 26, "matched_rule": { @@ -9013,10 +9013,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -12119,7 +12119,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 31, "end_line": 36, "matched_rule": { @@ -12243,7 +12243,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 29, "end_line": 34, "matched_rule": { @@ -12367,7 +12367,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 29, "end_line": 34, "matched_rule": { @@ -12646,10 +12646,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -13162,10 +13162,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 23, "end_line": 23, "matched_rule": { @@ -13254,10 +13254,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/summarycode/data/plugin_consolidate/zlib.json b/tests/summarycode/data/plugin_consolidate/zlib.json index 3ee5ceffc45..c3ebf3b4796 100644 --- a/tests/summarycode/data/plugin_consolidate/zlib.json +++ b/tests/summarycode/data/plugin_consolidate/zlib.json @@ -3857,10 +3857,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -3984,10 +3984,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4099,10 +4099,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4279,10 +4279,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4394,10 +4394,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4509,10 +4509,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4624,10 +4624,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4739,10 +4739,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4854,10 +4854,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 6, "matched_rule": { @@ -4969,10 +4969,10 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 6, "end_line": 25, "matched_rule": { @@ -8030,7 +8030,7 @@ "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style", "reference_url": "https://scancode-licensedb.aboutcode.org/mit-old-style", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 9, "end_line": 15, "matched_rule": { @@ -8177,10 +8177,10 @@ "is_unknown": false, "owner": "nexB", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/other-permissive", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 27, "matched_rule": { @@ -10294,7 +10294,7 @@ "text_url": "ftp://ftp.info-zip.org/pub/infozip/license.html", "reference_url": "https://scancode-licensedb.aboutcode.org/info-zip-2009-01", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 26, "end_line": 26, "matched_rule": { @@ -11050,10 +11050,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 5, "end_line": 5, "matched_rule": { @@ -13795,7 +13795,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 31, "end_line": 36, "matched_rule": { @@ -13917,7 +13917,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 29, "end_line": 34, "matched_rule": { @@ -14039,7 +14039,7 @@ "text_url": "http://www.ietf.org/rfc/rfc1952.txt", "reference_url": "https://scancode-licensedb.aboutcode.org/peter-deutsch-document", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 29, "end_line": 34, "matched_rule": { @@ -14282,10 +14282,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { @@ -14820,10 +14820,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 23, "end_line": 23, "matched_rule": { @@ -14910,10 +14910,10 @@ "is_unknown": false, "owner": "Unspecified", "homepage_url": "http://www.linfo.org/publicdomain.html", - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/public-domain", "spdx_license_key": null, - "spdx_url": "", + "spdx_url": null, "start_line": 2, "end_line": 2, "matched_rule": { diff --git a/tests/summarycode/data/tallies/full_tallies/tallies.expected.json b/tests/summarycode/data/tallies/full_tallies/tallies.expected.json index c72d3f1f8b7..b2a40d49cbc 100644 --- a/tests/summarycode/data/tallies/full_tallies/tallies.expected.json +++ b/tests/summarycode/data/tallies/full_tallies/tallies.expected.json @@ -7676,7 +7676,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/tallies/full_tallies/tallies_by_facet.expected.json b/tests/summarycode/data/tallies/full_tallies/tallies_by_facet.expected.json index bac737fbcde..1c0a5a1744b 100644 --- a/tests/summarycode/data/tallies/full_tallies/tallies_by_facet.expected.json +++ b/tests/summarycode/data/tallies/full_tallies/tallies_by_facet.expected.json @@ -8351,7 +8351,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/tallies/full_tallies/tallies_details.expected.json b/tests/summarycode/data/tallies/full_tallies/tallies_details.expected.json index 151887ad127..f6514ca1e07 100644 --- a/tests/summarycode/data/tallies/full_tallies/tallies_details.expected.json +++ b/tests/summarycode/data/tallies/full_tallies/tallies_details.expected.json @@ -8595,7 +8595,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/tallies/full_tallies/tallies_key_files-details.expected.json-lines b/tests/summarycode/data/tallies/full_tallies/tallies_key_files-details.expected.json-lines index 676b6c62884..4413f5b362e 100644 --- a/tests/summarycode/data/tallies/full_tallies/tallies_key_files-details.expected.json-lines +++ b/tests/summarycode/data/tallies/full_tallies/tallies_key_files-details.expected.json-lines @@ -2094,7 +2094,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", diff --git a/tests/summarycode/data/tallies/full_tallies/tallies_key_files.expected.json b/tests/summarycode/data/tallies/full_tallies/tallies_key_files.expected.json index 2d4fbc95e39..4a191c2c674 100644 --- a/tests/summarycode/data/tallies/full_tallies/tallies_key_files.expected.json +++ b/tests/summarycode/data/tallies/full_tallies/tallies_key_files.expected.json @@ -1418,7 +1418,7 @@ "is_unknown": false, "owner": "Dmitriy Anisimkov", "homepage_url": null, - "text_url": "", + "text_url": null, "reference_url": "https://scancode-licensedb.aboutcode.org/ada-linking-exception", "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.LICENSE", "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/ada-linking-exception.yml", From f201faaa6330c4ec17b32b6832520b207d52ca45 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 21 Oct 2022 22:40:18 +0530 Subject: [PATCH 40/43] Add --only-builtin falg for scancode-reindex-licenses Signed-off-by: Ayan Sinha Mahapatra --- src/licensedcode/cache.py | 51 ++++++++++++++++++++++++++++--------- src/licensedcode/reindex.py | 23 ++++++++++++++++- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 000accc8257..840df78b9db 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -6,9 +6,10 @@ # See https://github.com/nexB/scancode-toolkit for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click + import os import pickle +from shutil import rmtree import attr @@ -53,6 +54,7 @@ class LicenseCache: @staticmethod def load_or_build( + only_builtin=False, licensedcode_cache_dir=licensedcode_cache_dir, scancode_cache_dir=scancode_cache_dir, force=False, @@ -81,6 +83,9 @@ def load_or_build( directories using the same format that we use for licenses and rules. """ idx_cache_dir = os.path.join(licensedcode_cache_dir, LICENSE_INDEX_DIR) + if only_builtin: + rmtree(idx_cache_dir) + create_dir(idx_cache_dir) cache_file = os.path.join(idx_cache_dir, LICENSE_INDEX_FILENAME) @@ -120,14 +125,18 @@ def load_or_build( # rebuild all cached data (e.g. mostly the index) and cache it additional_directories = [] - plugin_directories = get_paths_to_installed_licenses_and_rules() - if plugin_directories: - additional_directories.extend(plugin_directories) - - # include installed licenses - if additional_directory: - # additional_directories is originally a tuple - additional_directories.append(additional_directory) + if only_builtin: + additional_directory = None + plugin_directories = [] + else: + plugin_directories = get_paths_to_installed_licenses_and_rules() + if plugin_directories: + additional_directories.extend(plugin_directories) + + # include installed licenses + if additional_directory: + # additional_directories is originally a tuple + additional_directories.append(additional_directory) additional_license_dirs = get_license_dirs(additional_dirs=additional_directories) validate_additional_license_data( @@ -355,7 +364,12 @@ def build_unknown_spdx_symbol(licenses_db=None): return LicenseSymbolLike(licenses_db['unknown-spdx']) -def get_cache(force=False, index_all_languages=False, additional_directory=None): +def get_cache( + only_builtin=False, + force=False, + index_all_languages=False, + additional_directory=None +): """ Return a LicenseCache either rebuilt, cached or loaded from disk. @@ -364,13 +378,19 @@ def get_cache(force=False, index_all_languages=False, additional_directory=None) texts and rules (the default) """ return populate_cache( + only_builtin=only_builtin, force=force, index_all_languages=index_all_languages, additional_directory=additional_directory, ) -def populate_cache(force=False, index_all_languages=False, additional_directory=None): +def populate_cache( + only_builtin=False, + force=False, + index_all_languages=False, + additional_directory=None +): """ Return, load or build and cache a LicenseCache. """ @@ -378,6 +398,7 @@ def populate_cache(force=False, index_all_languages=False, additional_directory= if force or not _LICENSE_CACHE: _LICENSE_CACHE = LicenseCache.load_or_build( + only_builtin=only_builtin, licensedcode_cache_dir=licensedcode_cache_dir, scancode_cache_dir=scancode_cache_dir, force=force, @@ -407,11 +428,17 @@ def load_cache_file(cache_file): raise Exception(msg) from e -def get_index(force=False, index_all_languages=False, additional_directory=None): +def get_index( + only_builtin=False, + force=False, + index_all_languages=False, + additional_directory=None +): """ Return and eventually build and cache a LicenseIndex. """ return get_cache( + only_builtin=only_builtin, force=force, index_all_languages=index_all_languages, additional_directory=additional_directory diff --git a/src/licensedcode/reindex.py b/src/licensedcode/reindex.py index 47dcc94b01e..a32e5a9233f 100644 --- a/src/licensedcode/reindex.py +++ b/src/licensedcode/reindex.py @@ -9,6 +9,8 @@ import click +from commoncode.cliutils import PluggableCommandLineOption + @click.command(name='scancode-reindex-licenses') @click.option( @@ -16,6 +18,17 @@ is_flag=True, help='[EXPERIMENTAL] Rebuild the license index including texts all ' 'languages (and not only English) and exit.', + cls=PluggableCommandLineOption, +) +@click.option( + '--only-builtin', + is_flag=True, + help='Rebuild the license index excluding any additional ' + 'license directory or additional license plugins which' + 'were added previously, i.e. with only builtin scancode ' + 'license and rules.', + conflicting_options=['additional_directory'], + cls=PluggableCommandLineOption, ) @click.option( '--additional-directory', @@ -23,9 +36,12 @@ metavar='DIR', help='Include this directory with additional custom licenses and license rules ' 'in the license detection index.', + conflicting_options=['only_builtin'], + cls=PluggableCommandLineOption, ) @click.help_option('-h', '--help') def reindex_licenses( + only_builtin, all_languages, additional_directory, *args, @@ -35,7 +51,12 @@ def reindex_licenses( from licensedcode.cache import get_index click.echo('Rebuilding the license index...') - get_index(force=True, index_all_languages=bool(all_languages), additional_directory=additional_directory) + get_index( + only_builtin=only_builtin, + force=True, + index_all_languages=bool(all_languages), + additional_directory=additional_directory + ) click.echo('Done.') From 095c8ede52559378af041ddd90c08ab7e7927ada Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 22 Oct 2022 01:17:04 +0530 Subject: [PATCH 41/43] Update docs for external licenses Signed-off-by: Ayan Sinha Mahapatra --- docs/source/cli-reference/core-options.rst | 68 ------------ docs/source/cli-reference/index.rst | 1 + docs/source/cli-reference/list-options.rst | 4 + docs/source/cli-reference/other-commands.rst | 102 ++++++++++++++++++ .../install_new_license_plugin.rst | 91 ++++++++++++---- docs/source/rst_snippets/core_options.rst | 10 -- .../additional_directory_is_temp.rst | 6 ++ .../rst_snippets/note_snippets/core_indep.rst | 4 - .../note_snippets/license_plugin_delete.rst | 7 ++ .../license_plugin_needs_reindex.rst | 5 + .../scancode-reindex-licenses.rst | 21 ++++ 11 files changed, 216 insertions(+), 103 deletions(-) create mode 100644 docs/source/cli-reference/other-commands.rst create mode 100644 docs/source/rst_snippets/note_snippets/additional_directory_is_temp.rst delete mode 100644 docs/source/rst_snippets/note_snippets/core_indep.rst create mode 100644 docs/source/rst_snippets/note_snippets/license_plugin_delete.rst create mode 100644 docs/source/rst_snippets/note_snippets/license_plugin_needs_reindex.rst create mode 100644 docs/source/rst_snippets/scancode-reindex-licenses.rst diff --git a/docs/source/cli-reference/core-options.rst b/docs/source/cli-reference/core-options.rst index 7bfc84e86b6..55e4501a1cc 100644 --- a/docs/source/cli-reference/core-options.rst +++ b/docs/source/cli-reference/core-options.rst @@ -69,74 +69,6 @@ Comparing Progress Message Options ---- -``--reindex-licenses`` Option ------------------------------ - - ScanCode maintains a license index to search for and detect licenses. When Scancode is - configured for the first time, a license index is built and used in every scan thereafter. - - This ``--reindex-licenses`` option rebuilds the license index. Running a scan with this option - displays the following message to the terminal in addition to what it normally shows:: - - Checking and rebuilding the license index... - - .. - [ToDo] Research and Write Better - -``--additional-license-directory`` Option ------------------------------------------ - - .. admonition:: Dependency - - The option ``--additional-license-directory`` requires the option ``-reindex--licenses``. - - The ``--additional-license-directory`` option allows the user to include additional directories - of licenses to use in license detection. - - This command only needs to be run once for each set of additional directories; in all subsequent - runs of Scancode with the same directories all the licenses in the directories will be cached. - - The directory structure should look something like this:: - - licenses/ - ├── privateLicense1/ - │ ├── license/ - │ │ ├── privateLicense1.LICENSE - │ │ └── privateLicense1.yml - │ └── rule/ - │ ├── privateLicense1.RULE - │ └── privateLicense1.yml - └── privateLicense2/ - ├── license/ - │ ├── privateLicense2.LICENSE - │ └── privateLicense2.yml - └── rule/ - ├── privateLicense2.RULE - └── privateLicense2.yml - - Here is an example of reindexing the license cache using the ``--additional-license-directory PATH`` option with a single directory. - Note that ``--reindex-licenses`` **must** come after ``--additional-license-directory``:: - - scancode --additional-license-directory /home/user/external_licenses/license1 --reindex-licenses - - You can also include multiple directories like so:: - - scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external2 --reindex-licenses - - If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``, - you can simply run scans after reindexing with those directories and they will be included. :: - - scancode -clpieu --json-pp output.json samples - - However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1`` - and ``home/user/external_licenses/external3``, you would need to reindex the license index with those directories as parameters. :: - - scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external3 --reindex-licenses - - .. - ----- - ``--from-json`` Option ---------------------- diff --git a/docs/source/cli-reference/index.rst b/docs/source/cli-reference/index.rst index c38f3df5433..c8b22e4151c 100644 --- a/docs/source/cli-reference/index.rst +++ b/docs/source/cli-reference/index.rst @@ -8,6 +8,7 @@ help-text-options list-options simple-examples + other-commands basic-options core-options output-format diff --git a/docs/source/cli-reference/list-options.rst b/docs/source/cli-reference/list-options.rst index f8e7f695400..e1bcf1cf9e2 100644 --- a/docs/source/cli-reference/list-options.rst +++ b/docs/source/cli-reference/list-options.rst @@ -30,6 +30,10 @@ available in the command line. ---- +.. include:: /rst_snippets/scancode-reindex-licenses.rst + +---- + .. include:: /rst_snippets/core_options.rst ---- diff --git a/docs/source/cli-reference/other-commands.rst b/docs/source/cli-reference/other-commands.rst new file mode 100644 index 00000000000..e2753ef2a5b --- /dev/null +++ b/docs/source/cli-reference/other-commands.rst @@ -0,0 +1,102 @@ +Other available CLIs +==================== + +.. _other_cli: + +---- + +.. include:: /rst_snippets/scancode-reindex-licenses.rst + +---- + +.. include:: /rst_snippets/extract.rst + +---- + +``scancode-reindex-licenses`` command +------------------------------------- + +ScanCode maintains a license index to search for and detect licenses. When Scancode is +configured for the first time, a license index is built and used in every scan thereafter. + +This ``scancode-reindex-licenses`` command rebuilds the license index. Running this command +displays the following message to the terminal:: + + Checking and rebuilding the license index... + +This has several CLI options as follows: + +``--additional-directory`` Option: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``--additional-directory`` option allows the user to include additional directories +of licenses to use in license detection. + +This command only needs to be run once for each set of additional directories, in all subsequent +runs of Scancode with the same directories all the licenses in the directories will be cached +and used in License detection. But reindexing removes these directories, if they aren't +reintroduced as additional directories. + +The directory structure should look something like this:: + + additional_license_directory/ + ├── licenses/ + │ ├── example-installed-1.LICENSE + │ └── example-installed-1.yaml + ├── rules/ + │ ├── example-installed-1.RULE + │ └── example-installed-1.yaml + +Here is an example of reindexing the license cache using the ``--additional-directory PATH`` option +with a single directory:: + + scancode-reindex-licenses --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/ + +You can also include multiple directories like so:: + + scancode-reindex-licenses --additional-directory /home/user/external_licenses/external1 --additional-directory /home/user/external_licenses/external2 + +If you want to continue running scans with ``/home/user/external_licenses/external1`` and +``/home/user/external_licenses/external2``, you can simply run scans after the command above +reindexing with those directories and they will be included. :: + + scancode -l --license-text --json-pp output.json samples + +However, if you wanted to run a scan with a new set of directories, such as +``home/user/external_licenses/external1`` and ``home/user/external_licenses/external3``, you would +need to reindex the license index with those directories as parameters:: + + scancode --additional-directory /home/user/external_licenses/external1 --additional-directory /home/user/external_licenses/external3 + +.. include:: /rst_snippets/note_snippets/additional_directory_is_temp.rst + + +.. note:: + + You can also install external licenses through a plugin for + better reproducibility and distribution of those license/rules + for use in conjunction with scancode-toolkit licenses. + See :ref:`install_new_license_plugin` + + +``--only-builtin`` Option: +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rebuild the license index excluding any additional license directory or additional +license plugins which were added previously, i.e. with only builtin scancode license and rules. + +This is applicable when there are additional license plugins installed already and you want to +reindex the licenses without these licenses from the additional plugins. + +.. note:: + + Running the ``--only-builtin`` command won't get rid of the installed license plugins, it + would just reindex without the licenses from these plugins for once. Another reindex afterwards + without this option would bring back the licenses from the plugins again in the index. + + +``--all-languages`` Option: +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rebuild the license index including texts all languages (and not only +English) and exit. This is an EXPERIMENTAL option. diff --git a/docs/source/how-to-guides/install_new_license_plugin.rst b/docs/source/how-to-guides/install_new_license_plugin.rst index cd3d1756f3f..28092e3e883 100644 --- a/docs/source/how-to-guides/install_new_license_plugin.rst +++ b/docs/source/how-to-guides/install_new_license_plugin.rst @@ -1,18 +1,25 @@ -.. _install_new_license_plugin: +.. _install_external_licenses: + +How to Install External Licenses to Use in License Detection +============================================================ -How to Install External License Plugins to Use in License Detection -=================================================================== +Users can install external licenses and rules in the form of: -Users can install external licenses and rules in the form of plugins. These -licenses and rules are then used in license detection. +1. reusable plugins +2. license directories -How to create a plugin containing external licenses and/or rules ----------------------------------------------------------------- +These licenses and rules are then used in license detection. + +.. _install_new_license_plugin: + +How to install a plugin containing external licenses and/or rules +----------------------------------------------------------------- To create a plugin with external licenses or rules, we must create a Python package containing the license and/or rule files. Python packages can have many different -file structures. You can find an example package in -``tests/licensedcode/data/example_external_licenses/licenses_to_install1``. +file structures. You can find an example package in: + +``tests/licensedcode/data/additional_licenses/additional_plugin_1``. This is the basic structure of the example plugin:: @@ -26,14 +33,11 @@ This is the basic structure of the example plugin:: │ │ ├── example-installed-1.RULE │ │ └── example-installed-1.yaml │ └── __init__.py - ├── gpl-1.0.LICENSE + ├── apache-2.0.LICENSE ├── MANIFEST.in ├── setup.cfg └── setup.py -Key points to note ------------------- - Entry points definition in ``setup.py`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -75,18 +79,27 @@ After creating this plugin, you can upload it to PyPI so that others can use it, leave it as a local directory. Installing and using the plugin -------------------------------- -To use the plugin in license detection, all you need to do is install it using ``pip``. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use the plugin in license detection, all you need to do is: + +1. Configure the scancode-toolkit virtualenv and activate. +2. Install the package with `pip` like the following: + ``pip install tests/licensedcode/data/additional_licenses/additional_plugin_2/`` +3. Reindex licenses using `scancode-reindex-licenses`. + +.. include:: /rst_snippets/note_snippets/license_plugin_needs_reindex.rst + Once it is installed, the contained licenses and rules will automatically be used in license detection assuming the plugin follows the correct directory structure conventions. Writing tests for new installed licenses ----------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Look at ``tests/licensedcode/data/example_external_licenses/licenses_to_install1`` to see an example of a plugin with tests. The tests are contained in the ``tests`` directory:: - licenses_to_install1/ + licenses_to_install1/ ├── src/ │ └── licenses_to_install1/ │ ├── licenses/ @@ -132,8 +145,44 @@ Then you can define a test class and call the ``build_tests`` method defined in The ``tests/data`` directory contains a pair of files for each license: a license text file and a YAML file specifying the expected license expressions from the test. -Finally, to run the test, do the following: +Finally, install the plugin and run the test: + +``pytest -vvs tests/test_detection_datadriven.py``. + +.. include:: /rst_snippets/note_snippets/license_plugin_delete.rst + +---- + +.. _add_new_license_directory: + +How to add external licenses and/or rules from a directory +---------------------------------------------------------- + +This is the basic structure of the example license directory:: + + additional_license_directory/ + ├── licenses/ + │ ├── example-installed-1.LICENSE + │ └── example-installed-1.yaml + ├── rules/ + │ ├── example-installed-1.RULE + │ └── example-installed-1.yaml + +Adding the licenses to the index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To add the licenses in the directory to the index, all you need to do is: + +1. Configure the scancode-toolkit virtualenv and activate. +2. Run ``scancode-reindex-licenses`` with: + + ``--additional-directory tests/licensedcode/data/additional_licenses/additional_dir/`` + +.. include:: /rst_snippets/note_snippets/additional_directory_is_temp.rst + + +Once the licenses/rules are in the index, they will automatically be used in license detection. + +---- -1. Create a virtual environment to install the package into. -2. Install the package using ``pip``, e.g. ``pip install ./licenses_to_install1``. -3. Run the tests, e.g. ``py.test tests/test_detection_datadriven.py``. +.. include:: /rst_snippets/scancode-reindex-licenses.rst diff --git a/docs/source/rst_snippets/core_options.rst b/docs/source/rst_snippets/core_options.rst index 3ed523415ab..734b0ad1cd4 100644 --- a/docs/source/rst_snippets/core_options.rst +++ b/docs/source/rst_snippets/core_options.rst @@ -11,14 +11,6 @@ All "Core" Scan Options --timeout FLOAT Stop scanning a file if scanning takes longer than a timeout in seconds. [Default: 120] ---reindex-licenses Force a check and possible reindexing of the - cached license index. - ---additional-license-directory PATH - - Include paths to directories containing additional licenses and rules to use - in license detection. This can be used multiple times for multiple directories. - --from-json Load codebase from an existing JSON scan --max-in-memory INTEGER Maximum number of files and directories scan @@ -33,5 +25,3 @@ All "Core" Scan Options including and below the starting point. INTEGER must be positive or zero for no limit. [Default: 0] - -.. include:: /rst_snippets/note_snippets/core_indep.rst diff --git a/docs/source/rst_snippets/note_snippets/additional_directory_is_temp.rst b/docs/source/rst_snippets/note_snippets/additional_directory_is_temp.rst new file mode 100644 index 00000000000..507eccb5821 --- /dev/null +++ b/docs/source/rst_snippets/note_snippets/additional_directory_is_temp.rst @@ -0,0 +1,6 @@ +.. note:: + + Adding licenses/rules from an additional directory is not permanent. + Another reindexing without the additional directory option would + just use the builtin scancode licenses and rules, and will not have + these additonal licenses/rules anymore. diff --git a/docs/source/rst_snippets/note_snippets/core_indep.rst b/docs/source/rst_snippets/note_snippets/core_indep.rst deleted file mode 100644 index 8fe7f93cdf3..00000000000 --- a/docs/source/rst_snippets/note_snippets/core_indep.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. note:: - - All the Core Options except for ``--additional-license-directory`` are independent options, i.e. - They don't depend on other options. diff --git a/docs/source/rst_snippets/note_snippets/license_plugin_delete.rst b/docs/source/rst_snippets/note_snippets/license_plugin_delete.rst new file mode 100644 index 00000000000..d0328499753 --- /dev/null +++ b/docs/source/rst_snippets/note_snippets/license_plugin_delete.rst @@ -0,0 +1,7 @@ +.. note:: + + Once you install a external license plugin, you have to reconfigure + scancode-toolkit (or use pip uninstall) to uninstall the plugin to + completely remove it. Otherwise using the --only-builtin option only + regenerates the index without the installed plugins, but another Reindex + would have the licenses/rules from the installed plugins. diff --git a/docs/source/rst_snippets/note_snippets/license_plugin_needs_reindex.rst b/docs/source/rst_snippets/note_snippets/license_plugin_needs_reindex.rst new file mode 100644 index 00000000000..51b46b8cbb1 --- /dev/null +++ b/docs/source/rst_snippets/note_snippets/license_plugin_needs_reindex.rst @@ -0,0 +1,5 @@ +.. note:: + + Installing the plugin will not add the licenses/rules to the + index automatically, they will be indexed only after running + `scancode-reindex-licenses`. diff --git a/docs/source/rst_snippets/scancode-reindex-licenses.rst b/docs/source/rst_snippets/scancode-reindex-licenses.rst new file mode 100644 index 00000000000..d6aae4bec98 --- /dev/null +++ b/docs/source/rst_snippets/scancode-reindex-licenses.rst @@ -0,0 +1,21 @@ +``scancode-reindex-licenses`` Usage +----------------------------------- + +Usage: ``scancode-reindex-licenses [OPTIONS]`` + +Reindex scancode licenses and exit + +Options +------- + + --all-languages [EXPERIMENTAL] Rebuild the license index + including texts all languages (and not only + English) and exit. + --only-builtin Rebuild the license index excluding any + additional license directory or additional + license plugins which were added previously, i.e. + with only builtin scancode license and rules. + --additional-directory DIR Include this directory with additional custom + licenses and license rules in the license + detection index. + -h, --help Shows the options and explanations. From 54fb102c29c8fbd65e4b93a4d0d7cd7c22b9ea53 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 28 Oct 2022 16:12:03 +0530 Subject: [PATCH 42/43] Refactor external licenses code Refactor according to the feedback. Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 +++- src/licensedcode/cache.py | 6 +++--- src/licensedcode/models.py | 2 ++ src/licensedcode/reindex.py | 2 +- .../{ => license_url}/license_url.expected.json | 4 ++-- .../plugin_license/license_url/{ => scan}/apache-1.0.txt | 0 tests/licensedcode/test_plugin_license.py | 4 ++-- 7 files changed, 13 insertions(+), 9 deletions(-) rename tests/licensedcode/data/plugin_license/{ => license_url}/license_url.expected.json (96%) rename tests/licensedcode/data/plugin_license/license_url/{ => scan}/apache-1.0.txt (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dc200a74566..c1d2fe47c5e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,7 +32,9 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven1.py \ --ignore=tests/licensedcode/test_detection_datadriven2.py \ --ignore=tests/licensedcode/test_detection_datadriven3.py \ - --ignore=tests/licensedcode/test_detection_datadriven4.py + --ignore=tests/licensedcode/test_detection_datadriven4.py \ + --ignore=tests/licensedcode/test_additional_license.py \ + tests/licensedcode license_datadriven1_2: | venv/bin/pytest -n 3 -vvs --test-suite=all \ diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 840df78b9db..4757152bf2d 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -72,9 +72,9 @@ def load_or_build( On the side, we load cached or build license db, SPDX symbols and other license-related data structures. - - If the cache exists, it is returned unless corrupted, ``force`` is True, or if we pass in additional - directories containing licenses that are not present in the existing cache. - - If the cache does not exist, a new index is built and cached. + - If the cache exists, it is returned unless corrupted. + - If ``force`` is True, or if the cache does not exist a new index is built + and cached. - If ``index_all_languages`` is True, include texts in all languages when building the license index. Otherwise, only include the English license texts and rules (the default) diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index f4fa7ce7e81..0b6ca63407d 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -444,6 +444,8 @@ def write(location, byte_string): if text: write(self.text_file(licenses_data_dir=licenses_data_dir), text.encode('utf-8')) + return self + def load(self, data_file, text_file): """ Populate license data from a YAML file stored in ``data_file`` and ``text_file``. diff --git a/src/licensedcode/reindex.py b/src/licensedcode/reindex.py index a32e5a9233f..8e3ab85385d 100644 --- a/src/licensedcode/reindex.py +++ b/src/licensedcode/reindex.py @@ -24,7 +24,7 @@ '--only-builtin', is_flag=True, help='Rebuild the license index excluding any additional ' - 'license directory or additional license plugins which' + 'license directory or additional license plugins which ' 'were added previously, i.e. with only builtin scancode ' 'license and rules.', conflicting_options=['additional_directory'], diff --git a/tests/licensedcode/data/plugin_license/license_url.expected.json b/tests/licensedcode/data/plugin_license/license_url/license_url.expected.json similarity index 96% rename from tests/licensedcode/data/plugin_license/license_url.expected.json rename to tests/licensedcode/data/plugin_license/license_url/license_url.expected.json index 1440419832d..d8c8e62c87e 100644 --- a/tests/licensedcode/data/plugin_license/license_url.expected.json +++ b/tests/licensedcode/data/plugin_license/license_url/license_url.expected.json @@ -1,7 +1,7 @@ { "files": [ { - "path": "license_url", + "path": "scan", "type": "directory", "licenses": [], "license_expressions": [], @@ -9,7 +9,7 @@ "scan_errors": [] }, { - "path": "license_url/apache-1.0.txt", + "path": "scan/apache-1.0.txt", "type": "file", "licenses": [ { diff --git a/tests/licensedcode/data/plugin_license/license_url/apache-1.0.txt b/tests/licensedcode/data/plugin_license/license_url/scan/apache-1.0.txt similarity index 100% rename from tests/licensedcode/data/plugin_license/license_url/apache-1.0.txt rename to tests/licensedcode/data/plugin_license/license_url/scan/apache-1.0.txt diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index 6d75a245375..d34e20c8cee 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -245,7 +245,7 @@ def test_reindex_licenses_works(): @pytest.mark.scanslow def test_scan_license_with_url_template(): - test_dir = test_env.get_test_loc('plugin_license/license_url', copy=True) + test_dir = test_env.get_test_loc('plugin_license/license_url/scan/', copy=True) result_file = test_env.get_temp_file('json') args = [ '--license', @@ -254,7 +254,7 @@ def test_scan_license_with_url_template(): '--json-pp', result_file, test_dir, ] - test_loc = test_env.get_test_loc('plugin_license/license_url.expected.json') + test_loc = test_env.get_test_loc('plugin_license/license_url/license_url.expected.json') run_scan_click(args) check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) From f2b1e13a10fe8e7b4863f22beba29691b4ada6c2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 28 Oct 2022 17:21:10 +0200 Subject: [PATCH 43/43] Improve CHANGELOG.rst Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7f4d4673c01..477590a071f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -57,6 +57,11 @@ License detection: These can be provided as a one off in a directory or packaged as a plugin for consistent reuse and deployment. +- There is a new "scancode-reindex-licenses" command that replace the + "scancode --reindex-licenses" command line option which has been + removed. This new command supports simpler reindexing using custom + license texts and license rules contributed by plugins or stored in an + additional directory. v31.2.1 - 2022-10-05 ----------------------------------