Add scancode evaluate script

The goal: check license offenders in pull request

This is similar to what astyle does in Travis. We get list of files being changed. Because scancode does not support list of files being scanned but rather a file or directory, we copy files to SCANCODE folder. Execute scancode license check in this folder and check for offenders.

The rules there are: code files must have a license and SDPX identifier. If they don't, we print these and ask for review.

It functions nicely there is just one workaround needed. SPDX is not always 100 percent correctly found, therefore we recheck file if no SPDX manually in the script. This proves to remove false positives.

Author: 0xc0170

.travis.yml

@@ -55,10 +55,24 @@ matrix:
     - <<: *basic-vm
       name: "license check"
       env: NAME=licence_check
+      language: python
+      python: 3.6.8 # scancode-toolkit v3.1.1 requires v3.6.8
+      install:
+        - pip install scancode-toolkit==3.1.1
+      before_script:
+        - mkdir -p SCANCODE
+        # Fetch remaining information needed for branch comparison
+        - git fetch --all --unshallow --tags
+        - git fetch origin "${TRAVIS_BRANCH}"
       script:
-        - |
-          ! grep --recursive --max-count=100 --ignore-case --exclude .travis.yml \
-          "gnu general\|gnu lesser\|lesser general\|public license"
+        # scancode does not support list of files, only one file or directory
+        # we use SCANCODE directory for all changed files (their copies with full tree)
+        - >-
+            git diff --name-only --diff-filter=d FETCH_HEAD..HEAD \
+              | ( grep '.\(c\|cpp\|h\|hpp\|py\)$' || true ) \
+              | while read file; do cp --parents "${file}" SCANCODE; done
+        - scancode -l --json-pp scancode.json SCANCODE
+        - python ./tools/test/travis-ci/scancode-evaluate.py -f scancode.json
 
     - <<: *basic-vm
       name: "include check"

tools/test/travis-ci/scancode-evaluate.py

@@ -0,0 +1,124 @@
+"""
+SPDX-License-Identifier: Apache-2.0
+
+Copyright (c) 2020 Arm Limited. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations 
+"""
+
+# Asumptions for this script: 
+# 1. directory_name is scanned directory.
+#  Files are copied to this directory with full tree. As result, if we find
+#  license offender, we can have full path (just scrape directory_name). We do this
+#  magic because scancode allows to scan directories/one file.
+# 2. SPDX and license text is a must for all code files
+
+import json
+import argparse
+import sys
+import os.path
+import logging
+import re
+
+userlog = logging.getLogger("scancode-evaluate")
+userlog.setLevel(logging.INFO)
+
+MISSING_LICENSE_TEXT = "Missing license header"
+MISSING_PERMISIVE_LICENSE_TEXT = "Non-permissive license"
+MISSING_SPDX_TEXT = "Missing SPDX license identifier"
+
+def license_check(directory_name, file):
+    """ Check licenses in the scancode json file for specified directory
+
+    This function does not verify if file exists, should be done prior the call.
+
+    Args:  
+    directory_name - where scancode was run, used to scrape this from paths   
+    file - scancode json output file (output from scancode --license --json-pp)
+
+    Returns:    
+    0 if success, -1 if any error in file licenses found
+    """
+
+    offenders = []
+    try:
+        # find all licenses in the files, must be licensed and permissive
+        with open(file, 'r') as scancode_output:
+            results = json.load(scancode_output)
+    except ValueError:
+        userlog.warning("JSON could not be decoded")
+        return -1
+
+    try:
+        for file in results['files']:
+            license_offender = {}
+            license_offender['file'] = file
+            # ignore directory, not relevant here
+            if license_offender['file']['type'] == 'directory':
+                continue
+            if not license_offender['file']['licenses']:
+                license_offender['reason'] = MISSING_LICENSE_TEXT
+                offenders.append(license_offender)
+                continue
+
+            found_spdx = False
+            for i in range(len(license_offender['file']['licenses'])):
+                if license_offender['file']['licenses'][i]['category'] != 'Permissive':
+                    license_offender['reason'] = MISSING_PERMISIVE_LICENSE_TEXT
+                    offenders.append(license_offender)
+                # find SPDX, it shall be one of licenses found
+                if license_offender['file']['licenses'][i]['matched_rule']['identifier'].find("spdx") != -1:
+                    found_spdx = True
+
+            if not found_spdx:
+                # Issue reported here https://github.com/nexB/scancode-toolkit/issues/1913
+                # We verify here if SPDX is not really there as SDPX is part of the license text
+                # scancode has some problems detecting it properly
+                with open(os.path.join(os.path.abspath(license_offender['file']['path'])), 'r') as spdx_file_check:
+                    filetext = spdx_file_check.read()
+                matches = re.findall("SPDX-License-Identifier:?", filetext)
+                if matches:
+                    continue
+                license_offender['reason'] = MISSING_SPDX_TEXT
+                offenders.append(license_offender)
+
+    except KeyError:
+        userlog.warning("Invalid scancode json file")
+        return -1
+
+    if offenders:
+        userlog.warning("Found files with missing license details, please review and fix")
+        for offender in offenders:
+            userlog.warning("File: " + offender['file']['path'][len(directory_name):] + " " + "reason: " + offender['reason'])
+        return -1
+    else:
+        return 0
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="License check.")
+    parser.add_argument('-f', '--file',
+                        help="scancode-toolkit output json file")
+    parser.add_argument('-d', '--directory_name', default="SCANCODE",
+                        help='Directory name where are files being checked')
+    return parser.parse_args()
+
+if __name__ == "__main__":
+
+    args = parse_args()
+
+    if args.file and os.path.isfile(args.file):
+        sys.exit(license_check(args.directory_name, args.file))
+    else:
+        userlog.warning("Could not find the scancode json file")
+        sys.exit(-1)