Merge branch 'develop' into rename-first-column-in-csv

.travis.yml

@@ -84,15 +84,24 @@ matrix:
       script: 
         - source bin/activate
         - cd docs
-        # This script tests the Sphinx Documentation build
-        # Also checks for broken links in the documentation
-        - ./scripts/sphinx_build_link_check.sh
-        # This script checks for documentation style errors
+        # Check that the Sphinx Documentation build minimally
+        - sphinx-build -E source build
+        # Check for documentation style errors
         - ./scripts/doc8_style_check.sh
       language: python
       python: "3.6"
       dist: bionic
 
+    - os: linux
+      sudo: required
+      script: 
+        # This checks .ABOUT file links using `about check`
+        - source bin/activate
+        - ./bin/about check thirdparty/
+      language: python
+      python: "3.6"
+      dist: bionic
+
     # Refer https://github.com/dkhamsing/awesome_bot#command-line for more info.
     - os: linux
       sudo: required
@@ -107,7 +116,7 @@ matrix:
         # This checks for broken links inside README.md files of plugins/plugins-builtin folders.
         - awesome_bot */**/*.md --allow-redirect --allow-dupe --skip-save-results
         # This checks for broken links inside Thirdparty .ABOUT files
-        - awesome_bot */*.ABOUT --allow-redirect --allow-dupe --skip-save-results
+        - awesome_bot thirdparty/*.ABOUT --allow-redirect --allow-dupe --skip-save-results
 
 addons:
   homebrew:

AUTHORS.rst

@@ -16,3 +16,6 @@ The following organizations or individuals have contributed to ScanCode:
 - Tushar Mittal @techytushar
 - Martin Petkov @MartinPetkov
 - Mrinal Paliwal @mriiinal
+- Mankaran Singh @MankaranSingh
+- Ravi Jain @JRavi2
+- Tushar Goel @TG1999

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+Outputs:
+ - The "headers" attribute in JSON outputs now contains a 'duration' field. #1942
+
+OS support:
+ - Add 10.15 macOS tests to azure-pipelines
+
 v3.0.2 (2019-02-15)
 -------------------
 

README.rst

@@ -162,7 +162,7 @@ License
 =======
 
 * Apache-2.0 with an acknowledgement required to accompany the scan output.
-* Public domain CC-0 for reference datasets.
+* Public domain CC0-1.0 for reference datasets.
 * Multiple licenses (GPL2/3, LGPL, MIT, BSD, etc.) for third-party components.
 
 See the NOTICE file and the .ABOUT files that document the origin and license of

azure-pipelines.yml

@@ -83,21 +83,21 @@ jobs:
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1013_py27
-          image_name: macos-10.13
+          job_name: macos1014_py27
+          image_name: macos-10.14
           python_versions: ['2.7']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
               scancode: bin/py.test -vvs --reruns=3 tests/scancode
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1014_py27
-          image_name: macos-10.14
+          job_name: macos1015_py27
+          image_name: macos-10.15
           python_versions: ['2.7']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
-              scancode: bin/py.test -vvs --reruns=3 tests/scancode
+              scancode: bin/py.test -vvs --reruns=3 tests/scancode          
 
 
 ################################################################################
@@ -236,19 +236,19 @@ jobs:
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos_1013_py36
-          image_name: macos-10.13
+          job_name: macos1014_py36
+          image_name: macos-10.14
           python_versions: ['3.6']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1014_py36
-          image_name: macos-10.14
+          job_name: macos1015_py36
+          image_name: macos-10.15
           python_versions: ['3.6']
           test_suites:
-              all: bin/py.test -n 2 -vvs --reruns=3
+              all: bin/py.test -n 2 -vvs --reruns=3          
 
     - template: etc/ci/azure-win.yml
       parameters:

docs/source/getting-started/home.rst

@@ -20,7 +20,7 @@ ScanCode tries to address this issue by offering:
 - Your choice of JSON or other output formats (SPDX, HTML, CSV) for integration with other tools
 - ScanCode workbench for Visualization
 - Well-tested, easy to hack, and well-documented code
-- Release of the code and reference data under attribution licenses (Apache 2.0 and CC-BY-1.0)
+- Release of the code and reference data under attribution licenses (Apache 2.0 and CC0-1.0)
 - Plugin System for easily adding new Functionality to Scans.
 - Python 3 Unicode Capabilities for better supporting users from 100+ languages.
 - Extensive Documentation Support.

docs/source/getting-started/whats-new.rst

@@ -49,7 +49,7 @@ deprecated hereafter. The new documentation has improved support in terms of:
 #. Consistent with the Latest Version
 #. Command Line Interface Reference
 #. Getting Started Support for Newcomers
-#. Restructured to fit into the recommended `4 Category Doc Format <https://www.divio.com/blog/documentation/>`_
+#. Restructured to fit into the recommended `4 Category Doc Format <https://docs.divio.com/en/latest/>`_
 #. Updated Tutorials/How To's
 #. Updated Plugin Support
 

docs/source/license.rst

@@ -3,4 +3,4 @@ License
 
 ScanCode Toolkit includes documents that are dedicated to the Public Domain using the
 Creative Commons CC0 1.0 Universal (CC0 1.0) Public Domain Dedication:
-http://creativecommons.org/publicdomain/zero/1.0/
+https://creativecommons.org/publicdomain/zero/1.0/

docs/source/misc/faq.rst

@@ -37,7 +37,7 @@ crafted regex):
   a grammar; it has a few thousand tests.
 - licenses and copyrights are detected in texts and binaries
 
-Licensecheck (available here for reference: /https://metacpan.org/release/App-Licensecheck )
+Licensecheck (available here for reference: /https://metacpan.org/pod/App::Licensecheck )
 is a Perl script using hand-crafted regex patterns to find typical copyright statements and
 about 50 common licenses. There are about 50 license detection tests.
 

etc/scripts/buildrules.py

@@ -184,19 +184,23 @@ def cli(licenses_file):
 
     print()
     for rule in rules_data:
+        is_negative = rule.data.get('is_negative')
+        is_false_positive = rule.data.get('is_false_positive')
         existing = rule_exists(rule.text)
-        if existing:
-            print('Skipping existing rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
+        if existing and not is_negative:
+            print('Skipping existing non-negative rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
             continue
-
-        if rule.data.get('is_negative'):
+        
+        if is_negative:
             base_name = 'not-a-license'
         else:
             license_expression = rule.data.get('license_expression')
             if not license_expression:
                 raise Exception('Missing license_expression for text:', rule)
             licensing.parse(license_expression, validate=True, simple=True)
             base_name = license_expression
+            if is_false_positive:
+                base_name = 'false-positive_' + base_name
 
         base_loc = find_rule_base_loc(base_name)
 
@@ -218,8 +222,8 @@ def cli(licenses_file):
         else:
             rules_tokens.add(rule_tokens)
             rulerec.dump()
-            models.update_ignorables(rulerec, verbose=True)
-            print('Rule added:', rulerec.identifier)
+            models.update_ignorables(rulerec, verbose=False)
+            print('Rule added:', 'file://' + rulerec.data_file, '\n', 'file://' + rulerec.text_file,)
 
 
 if __name__ == '__main__':

src/commoncode/hash.py

@@ -103,7 +103,7 @@ def __init__(self, msg=None):
         self.h = msg and self._compute(msg) or None
 
     def _compute(self, msg):
-        return hashlib.sha1(b'blob ' + bytes(len(msg)) + b'\0' + msg).digest()
+        return hashlib.sha1(b'blob ' + str(len(msg)).encode('ascii') + b'\0' + msg).digest()
 
     def digest(self):
         return bytes(self.h)

src/extractcode/extract.py

@@ -113,7 +113,7 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_or
        is True.
 
     If `recurse` is True, extract recursively archives nested inside other
-    archives If `recurse` is false, then do not extract further an already
+    archives. If `recurse` is false, then do not extract further an already
     extracted archive identified by the corresponding extract suffix location.
 
     If `replace_originals` is True, the extracted archives are replaced by the
@@ -123,9 +123,16 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_or
     if recurse and a nested archive is found, it is extracted to full depth
     first before resuming the file system walk.
     """
-    events = list(extract_files(location, kinds, recurse))
+    processed_events = []
+    processed_events_append = processed_events.append
+    for event in extract_files(location, kinds, recurse):
+        yield event
+        if replace_originals:
+            processed_events_append(event)
+
+    # move files around
     if replace_originals:
-        for xevent in reversed(events):
+        for xevent in reversed(processed_events):
             if xevent.done:
                 source = xevent.source
                 target = xevent.target
@@ -134,12 +141,22 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_or
                 fileutils.delete(source)
                 fileutils.copytree(target, source)
                 fileutils.delete(target)
-    return events
+
 
 def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
+    """
+    Extract the files found at `location`.
+
+    Extract only archives of a kind listed in the `kinds` kind tuple.
+
+    If `recurse` is True, extract recursively archives nested inside other
+    archives. If `recurse` is false, then do not extract further an already
+    extracted archive identified by the corresponding extract suffix location.
+    """
     ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={})
     if TRACE:
         logger.debug('extract:start: %(location)r  recurse: %(recurse)r\n' % locals())
+
     abs_location = abspath(expanduser(location))
     for top, dirs, files in fileutils.walk(abs_location, ignored):
         if TRACE:
@@ -192,8 +209,8 @@ def extract_file(location, target, kinds=extractcode.default_kinds, verbose=Fals
     extractor = archive.get_extractor(location, kinds)
     if TRACE:
         logger.debug('extract_file: extractor: for: %(location)r with kinds: %(kinds)r : ' % locals()
-                     + getattr(extractor, '__module__', '')
-                     + '.' + getattr(extractor, '__name__', ''))
+                     +getattr(extractor, '__module__', '')
+                     +'.' + getattr(extractor, '__name__', ''))
     if extractor:
         yield ExtractEvent(location, target, done=False, warnings=[], errors=[])
         try:

src/licensedcode/data/licenses/agpl-generic-additional-terms.yml

@@ -0,0 +1,9 @@
+key: agpl-generic-additional-terms
+short_name: AGPL Generic Additional Terms
+name: AGPL Generic Additional Terms
+is_exception: yes
+category: Copyleft
+owner: Unspecified
+notes: this is a generic entry for rare one-off AGPL extra license terms.
+ These are typically additional terms under section 7 of the AGPL-3.0
+

src/licensedcode/data/licenses/bsd-3-clause-devine.LICENSE

@@ -0,0 +1,25 @@
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  * Redistributions of source code _must_ retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form may or may not reproduce the above
+    copyright notice, this list of conditions and the following
+    disclaimer in the documentation and/or other materials provided
+    with the distribution.
+  * Neither the name of XySSL nor the names of its contributors may be
+    used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/bsd-3-clause-devine.yml

@@ -0,0 +1,9 @@
+key: bsd-3-clause-devine
+short_name: BSD 3-Clause Devine
+name: BSD 3-Clause Devine
+category: Permissive
+owner: Christophe Devine
+notes: The second clause for binaries is unique as it is optional
+ and no longer a condition per this statement "Redistributions in binary form
+ may or may not reproduce"
+ Seen in AES code from GPL Ghostscript, originally from XyzSSL

src/licensedcode/data/licenses/bsd-atmel.LICENSE

@@ -0,0 +1,19 @@
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice,
+this list of conditions and the disclaimer below.
+
+- Author's name may not be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+DISCLAIMER: THIS SOFTWARE IS PROVIDED BY AUTHOR "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE
+DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/bsd-atmel.yml

@@ -0,0 +1,8 @@
+key: bsd-atmel
+short_name: BSD Atmel License
+name: BSD Atmel License
+category: Permissive
+owner: Atmel
+homepage_url: https://github.com/atmelcorp/atmel-software-package/blob/d11d2a558fb5bd017aa15a9a78fb237ab972f29c/LICENSE.md
+minimum_coverage: 90
+notes: this is a bsd with clause 1 and 3

src/licensedcode/data/licenses/bsd-axis-nomod.yml

@@ -3,6 +3,7 @@ short_name: BSD-Axis without modification
 name: BSD-Axis without modification
 category: Permissive
 owner: Unspecified
+minimum_coverage: 95
 notes: This is a variant composed of clause 1 and 3 of a BSD-Modified found in the Linux kernel
     And this extra variant has the words "without modification" added at the end of clause 1
     either with or without a comma

src/licensedcode/data/licenses/bsd-axis.LICENSE

@@ -9,15 +9,15 @@ are met:
    contributors may be used to endorse or promote products derived
    from this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND ITS CONTRIBUTORS
-``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDERS OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE. 
+POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/bsd-axis.yml

@@ -4,3 +4,5 @@ name: BSD-Axis
 category: Permissive
 owner: Axis Communications
 notes: This is a variant composed of clause 1 and 3 of a BSD-Modified found in the Linux kernel
+ This is now replaced by the bsd-source-code license.
+is_deprecated: yes

src/licensedcode/data/licenses/bsd-new-derivative.yml

@@ -5,3 +5,4 @@ category: Permissive
 owner: libpcap
 homepage_url: http://www.tcpdump.org/sniffex.c
 notes: this is a rather rare variant of the bsd-new found in tcpdump
+minimum_coverage: 70