Update license detection

CHANGELOG.rst

@@ -37,6 +37,10 @@ License scanning:
  - Add new command line option to filter ignorable copyrights when included
    in licenses.
 
+ - Add new and improved license detection rules.
+   Thank you to:
+    - Sebastian Thomas @sebathomas
+    - Till Jaeger @LeChasseur 
 
 
 v21.3.31
@@ -93,6 +97,9 @@ Copyright scanning:
  - Allow calling copyright detection from text lines to ease integration
    Thank you to Jelmer Vernooij @jelmer
 
+ - Fixed copyright truncation bug
+   Thank you to Akanksha Garg @akugarg
+
 
 Package scanning:
 ~~~~~~~~~~~~~~~~~

azure-pipelines.yml

@@ -61,15 +61,35 @@ jobs:
                 bin/pytest -n 3 -vvs --test-suite=all \
                   tests/licensedcode/test_detection_datadriven_external.py
 
-              license_validate1: |
+              license_validate_basic: |
                 bin/pytest -n 3 -vvs --test-suite=validate \
                   tests/licensedcode/test_detection_validate.py \
                   -k TestValidateLicenseBasic
 
-              license_validate2: |
+              license_validate_extended_1: |
                 bin/pytest -n 3 -vvs --test-suite=validate \
                   tests/licensedcode/test_detection_validate.py \
-                  -k TestValidateLicenseExtended
+                  -k TestValidateLicenseExtended1
+
+              license_validate_extended_2: |
+                bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseExtended2
+
+              license_validate_extended_3: |
+                bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseExtended3
+
+              license_validate_extended_4: |
+                bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseExtended4
+
+              license_validate_extended_5: |
+                bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseExtended5
 
               license_cache: |
                 bin/pytest -n 3 -vvs --test-suite=all \

etc/scripts/licenses/gen_spdx_lists_fp.py

@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import click
+
+from licensedcode.tokenize import ngrams
+
+import synclic
+
+"""
+A script to generate false-positive license detection rules from lists of SPDX
+licenses.
+
+Common license detection tools use list of SPDX licenses ids to support their operations.
+As a result, we get a lot of matched licenses and in these cases, these are false positives.
+
+Here we fetch all released SPDX licenses lists and generate false positives
+using these approaches to have a reasonable set of combinations of license ids
+as found in the wild:
+
+1. For each SPDX license list release, we consider these lists:
+ - all IDs
+ - all non-deprecated IDs
+ - all licenses
+ - all non-deprecated licenses
+ - all exceptions
+ - all non-deprecated exceptions
+
+We generate lists of ids only and list of ids and name
+
+2. For each of these lists we sort them:
+ - respective case
+ - ignoring case
+
+3. for each of these sorted list we collect sub-sequences of 6 license, one
+  per line and generate a false positive RULE from that.
+
+If a RULE already exists, it will be skipped.
+"""
+
+TRACE = False
+
+template = '''----------------------------------------
+is_false_positive: yes
+notes: a sequence of SPDX license ids and names is not a license
+---
+{}
+'''
+
+
+@click.command()
+@click.argument(
+    'license_dir', type=click.Path(), metavar='DIR')
+
+@click.argument(
+    # 'A buildrules-formatted file used to generate new licenses rules.')
+    'output', type=click.Path(), metavar='FILE')
+
+@click.option(
+    '--commitish', type=str, default=None,
+    help='An optional commitish to use for SPDX license data instead of the latest release.')
+
+@click.option(
+    # 'A buildrules-formatted file used to generate new licenses rules.')
+    '--from-list', default=None, type=click.Path(), metavar='LIST_FILE',
+    help='Use file with a list of entries to ignore instead')
+
+@click.option(
+    '-t', '--trace', is_flag=True, default=False,
+    help='Print execution trace.')
+
+@click.help_option('-h', '--help')
+def cli(license_dir, output, commitish=None, from_list=None, trace=False):
+    """
+    Generate ScanCode false-positive license detection rules from lists of SPDX
+    license. Save these in FILE for use with buildrules.
+
+    the `spdx` directory is used as a temp store for fetched SPDX licenses.
+    """
+    global TRACE
+    TRACE = trace
+
+    if not from_list:
+        spdx_source = synclic.SpdxSource(external_base_dir=license_dir)
+
+        spdx_by_key = spdx_source.get_licenses(
+            commitish=commitish,
+            skip_oddities=False,
+        )
+
+        all_licenses_and_exceptions = []
+        all_licenses_and_exceptions_non_deprecated = []
+        licenses = []
+        exceptions = []
+        licenses_non_deprecated = []
+        exceptions_non_deprecated = []
+
+        lists_of_licenses = [
+            all_licenses_and_exceptions,
+            all_licenses_and_exceptions_non_deprecated,
+            licenses,
+            exceptions,
+            licenses_non_deprecated,
+            exceptions_non_deprecated,
+        ]
+
+        for lspdx in spdx_by_key.values():
+            all_licenses_and_exceptions.append(lspdx)
+            is_deprecated = lspdx.is_deprecated
+            if not is_deprecated:
+                all_licenses_and_exceptions_non_deprecated.append(lspdx)
+            if lspdx.is_exception:
+                exceptions.append(lspdx)
+                if not is_deprecated:
+                    exceptions_non_deprecated.append(lspdx)
+            else:
+                licenses.append(lspdx)
+                if not is_deprecated:
+                    licenses_non_deprecated.append(lspdx)
+
+        lists_of_sorted_licenses = []
+        for lic_list in lists_of_licenses:
+            sorted_case_sensitive = sorted(lic_list, key=lambda x: x.spdx_license_key)
+
+            as_ids = [l.spdx_license_key for l in sorted_case_sensitive]
+            lists_of_sorted_licenses.append(as_ids)
+
+            as_id_names = [f'{l.spdx_license_key}  {l.name}' for l in sorted_case_sensitive]
+            lists_of_sorted_licenses.append(as_id_names)
+
+            sorted_case_insensitive = sorted(lic_list, key=lambda x: x.spdx_license_key.lower())
+            as_ids = [l.spdx_license_key for l in sorted_case_insensitive]
+            lists_of_sorted_licenses.append(as_ids)
+
+            as_id_names = [f'{l.spdx_license_key}  {l.name}' for l in sorted_case_insensitive]
+            lists_of_sorted_licenses.append(as_id_names)
+
+    else:
+        with open(from_list) as inp:
+            lists_of_sorted_licenses = [inp.read().splitlines(False)]
+
+    with open(output, 'w') as o:
+        for lic_list in lists_of_sorted_licenses:
+            write_ngrams(texts=lic_list, output=o)
+
+        o.write('----------------------------------------\n')
+
+
+def write_ngrams(texts, output, _seen=set(), ngram_length=6):
+    """
+    Write the texts list as ngrams to the output file-like object.
+    """
+    for text in ['\n'.join(ngs) for ngs in ngrams(texts, ngram_length=ngram_length)]:
+        if text in _seen:
+            continue
+        _seen.add(text)
+        output.write(template.format(text))
+
+
+if __name__ == '__main__':
+    cli()

etc/scripts/licenses/synclic.py

@@ -162,15 +162,15 @@ def __init__(self, external_base_dir):
         if not exists(self.new_dir):
             mkdir(self.new_dir)
 
-    def get_licenses(self, scancode_licenses, **kwargs):
+    def get_licenses(self, scancode_licenses=None, **kwargs):
         """
         Return a mapping of key -> ScanCode License objects either fetched
         externally or loaded from the existing `self.original_dir`
         """
         print('Fetching and storing external licenses in:', self.original_dir)
 
         licenses = []
-        for lic, text in self.fetch_licenses(scancode_licenses, **kwargs):
+        for lic, text in self.fetch_licenses(scancode_licenses=scancode_licenses, **kwargs):
             try:
                 with io.open(lic.text_file, 'w', encoding='utf-8')as tf:
                     tf.write(text)
@@ -336,10 +336,19 @@ class SpdxSource(ExternalLicensesSource):
         'notes',
     )
 
-    def fetch_licenses(self, scancode_licenses, commitish=None, from_repo=SPDX_DEFAULT_REPO):
+    def fetch_licenses(
+        self,
+        scancode_licenses=None,
+        commitish=None,
+        skip_oddities=True,
+        from_repo=SPDX_DEFAULT_REPO,
+    ):
         """
-        Yield License objects fetched from the latest SPDX license list.
-        Use the latest tagged version or the `commitish` is provided.
+        Yield License objects fetched from the latest SPDX license list. Use the
+        latest tagged version or the `commitish` if provided.
+        If skip_oddities is True, some oddities are skipped or handled
+        specially, such as licenses with a trailing + or foreign language
+        licenses.
         """
         if not commitish:
             # get latest tag
@@ -361,32 +370,40 @@ def fetch_licenses(self, scancode_licenses, commitish=None, from_repo=SPDX_DEFAU
                 and ('/json/details/' in path or '/json/exceptions/' in path)):
                     continue
                 if TRACE_FETCH: print('Loading license:', path)
-                if path.endswith('+.json'):
+                if skip_oddities and path.endswith('+.json'):
                     # Skip the old plus licenses. We use them in
                     # ScanCode, but they are deprecated in SPDX.
                     continue
                 details = json.loads(archive.read(path))
-                lic = self.build_license(details, scancode_licenses)
+                lic = self.build_license(
+                    mapping=details,
+                    scancode_licenses=scancode_licenses,
+                    skip_oddities=skip_oddities,
+                )
+
                 if lic:
                     yield lic
 
-    def build_license(self, mapping, scancode_licenses):
+    def build_license(self, mapping, skip_oddities=True, scancode_licenses=None):
         """
         Return a ScanCode License object built from an SPDX license mapping.
+        If skip_oddities is True, some oddities are skipped or handled
+        specially, such as licenses with a trailing + or foreign language
+        licenses.
         """
         spdx_license_key = mapping.get('licenseId') or mapping.get('licenseExceptionId')
         assert spdx_license_key
         spdx_license_key = spdx_license_key.strip()
         key = spdx_license_key.lower()
 
         # TODO: Not yet available in ScanCode
-        is_foreign = key in scancode_licenses.non_english_by_spdx_key
-        if is_foreign:
+        is_foreign = scancode_licenses and key in scancode_licenses.non_english_by_spdx_key
+        if skip_oddities and is_foreign:
             if TRACE: print('Skipping NON-english license FOR NOW:', key)
             return
 
         # these keys have a complicated history
-        if key in set([
+        if skip_oddities and key in set([
             'gpl-1.0', 'gpl-2.0', 'gpl-3.0',
             'lgpl-2.0', 'lgpl-2.1', 'lgpl-3.0',
             'agpl-1.0', 'agpl-2.0', 'agpl-3.0',
@@ -399,7 +416,7 @@ def build_license(self, mapping, scancode_licenses):
             return
 
         deprecated = mapping.get('isDeprecatedLicenseId', False)
-        if deprecated:
+        if skip_oddities and deprecated:
             # we use concrete keys for some plus/or later versions for
             # simplicity and override SPDX deprecation for these
             if key.endswith('+'):

src/licensedcode/data/rules/apache-2.0_741.RULE

@@ -0,0 +1,4 @@
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
+// NOTICE file corresponding to the section 4d of The Apache License,
+// Version 2.0,

src/licensedcode/data/rules/apache-2.0_741.yml

@@ -0,0 +1,7 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 100
+ignorable_authors:
+    - The Apache Software Foundation (http://www.apache.org/)
+ignorable_urls:
+    - http://www.apache.org/

src/licensedcode/data/rules/apache-2.0_762.RULE

@@ -0,0 +1 @@
+is Apache-licensed.

src/licensedcode/data/rules/apache-2.0_762.yml

@@ -0,0 +1,3 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 95

src/licensedcode/data/rules/apache-2.0_768.RULE

@@ -0,0 +1,4 @@
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+Includes software from other Apache Software Foundation projects,

src/licensedcode/data/rules/apache-2.0_768.yml

@@ -0,0 +1,5 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 95
+ignorable_urls:
+    - http://www.apache.org/

src/licensedcode/data/rules/apache-2.0_769.RULE

@@ -0,0 +1 @@
+Includes software from other Apache Software Foundation projects,

src/licensedcode/data/rules/apache-2.0_769.yml

@@ -0,0 +1,3 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 95

src/licensedcode/data/rules/apache-2.0_778.RULE

@@ -0,0 +1 @@
+Apache-licensed.

src/licensedcode/data/rules/apache-2.0_778.yml

@@ -0,0 +1,3 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 95

src/licensedcode/data/rules/apache-2.0_779.RULE

@@ -0,0 +1,3 @@
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
+See the LICENSE.txt for more details.

src/licensedcode/data/rules/apache-2.0_779.yml

@@ -0,0 +1,9 @@
+license_expression: apache-2.0
+is_license_notice: yes
+relevance: 95
+referenced_filenames:
+    - LICENSE.txt
+ignorable_authors:
+    - The Apache Software Foundation (http://www.apache.org/)
+ignorable_urls:
+    - http://www.apache.org/

src/licensedcode/data/rules/apache-2.0_785.RULE

@@ -0,0 +1,5 @@
+License
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+Same license text as listed above