diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 10a4ac09998..a3ea2eaca99 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,25 +15,39 @@ jobs: bin/py.test -n 2 -vvs --reruns=3 --test-suite=all \ --ignore=tests/scancode \ --ignore=tests/cluecode \ - --ignore=tests/licensedcode + --ignore=tests/licensedcode \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" - scancode: bin/py.test --reruns=3 -vvs --test-suite=all tests/scancode + scancode: | + bin/py.test --reruns=3 -vvs --test-suite=all tests/scancode \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" - cluecode: bin/py.test -n 2 --reruns=3 -vvs --test-suite=all tests/cluecode + cluecode: | + bin/py.test -n 2 --reruns=3 -vvs --test-suite=all tests/cluecode \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" license_base: | bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ --ignore=tests/licensedcode/test_zzzz_cache.py \ --ignore=tests/licensedcode/test_detection_datadriven1.py \ - tests/licensedcode + tests/licensedcode \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" license_main: | bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/licensedcode/test_detection_datadriven1.py + tests/licensedcode/test_detection_datadriven1.py \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" license_cache: | bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/licensedcode/test_zzzz_cache.py + tests/licensedcode/test_zzzz_cache.py \ + --cov=src --cov-report=term --cov-report=xml + bin/codecov --token "$CODECOV_TOKEN" - template: etc/ci/azure-win.yml parameters: diff --git a/src/scancode/cli.py b/src/scancode/cli.py index f4078f47c03..4b0857fd2c8 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -483,7 +483,6 @@ def scancode(ctx, input, # NOQA Other **kwargs are passed down to plugins as CommandOption indirectly through Click context machinery. """ - success = False try: # Validate CLI UI options dependencies and other CLI-specific inits @@ -513,6 +512,12 @@ def scancode(ctx, input, # NOQA echo_func=echo_stderr, *args, **kwargs) + # check for updates + from scancode.outdated import check_scancode_version + outdated = check_scancode_version() + if not quiet and outdated: + echo_stderr(outdated, fg='yellow') + except click.UsageError as e: # this will exit raise e @@ -1180,7 +1185,7 @@ def scan_codebase(codebase, scanners, processes=1, timeout=DEFAULT_TIMEOUT, location, rid, scan_errors, scan_time, scan_result, scan_timings = scans.next() else: location, rid, scan_errors, scan_time, scan_result, scan_timings = next(scans) - + if TRACE_DEEP: logger_debug( 'scan_codebase: location:', location, 'results:', scan_result) @@ -1253,7 +1258,7 @@ def terminate_pool(pool): def terminate_pool_with_backoff(pool, number_of_trials=3): - # try a few times to terminate, + # try a few times to terminate, for trial in range(number_of_trials, 1): try: pool.terminate() diff --git a/src/scancode/outdated.ABOUT b/src/scancode/outdated.ABOUT new file mode 100644 index 00000000000..5ecc8b84219 --- /dev/null +++ b/src/scancode/outdated.ABOUT @@ -0,0 +1,8 @@ +about_resource: outdated.py +name: pip +version: 9974f2 +download_url: https://raw.githubusercontent.com/pypa/pip/9974f274e1c86fb556b5d51dcfd2d4dc637b7b67/pip/utils/outdated.py +license: mit +notice_file: outdated.NOTICE +license_text_file: outdated.LICENSE +notes: this code was derived and heavily modified from pip/utils/outdated.py. \ No newline at end of file diff --git a/src/scancode/outdated.LICENSE b/src/scancode/outdated.LICENSE new file mode 100644 index 00000000000..33b8cdc323a --- /dev/null +++ b/src/scancode/outdated.LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2008-2014 The pip developers (see outdated.NOTICE file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/src/scancode/outdated.NOTICE b/src/scancode/outdated.NOTICE new file mode 100644 index 00000000000..edf2fdbd94b --- /dev/null +++ b/src/scancode/outdated.NOTICE @@ -0,0 +1,213 @@ +The following authors have contributed to pip: +Adam Wentz +Aleks Bunin +Alex Gaynor +Alex Grönholm +Alex Morega +Alexandre Conrad +Alli +Anatoly Techtonik +Andrei Geacar +Andrey Bulgakov +Anrs Hu +Anton Patrushev +Antonio Alvarado Hernandez +Antti Kaihola +Armin Ronacher +Ashley Manton +Baptiste Mispelon +Ben Darnell +Ben Rosser +Bence Nagy +Berker Peksag +Bernardo B. Marques +Bogdan Opanchuk +Brad Erickson +Bradley Ayers +Brian Rosner +Bruno Renié +Buck Golemon +Bussonnier Matthias +Carl Meyer +Chris Brinker +Chris Jerdonek +Chris McDonough +Chris Wolfe +Christian Oudard +Clark Boylan +Clay McClure +Cody Soyland +Cory Benfield +Craig Kerstiens +Cristian Sorinel +Dan Savilonis +Dan Sully +Daniel Collins +Daniel Hahler +Daniel Holth +Daniel Jost +Daniele Procida +Dav Clark +Dave Abrahams +David Aguilar +David Black +David Evans +David Pursehouse +David Wales +Davidovich +derwolfe +Dmitry Gladkov +Donald Stufft +Dongweiming +Dwayne Bailey +Endoh Takanao +enoch +Eric Gillingham +Eric Hanchrow +Erik M. Bray +Eugene Vereshchagin +Florian Briand +Francesco +Gabriel de Perthuis +Garry Polley +Geoffrey Lehée +George Song +Georgi Valkov +gizmoguy1 +Guilherme Espada +Guy Rozendorn +Herbert Pfennig +Hsiaoming Yang +Hugo Lopes Tavares +Hynek Schlawack +Ian Bicking +Ian Cordasco +Igor Sobreira +Ilya Baryshev +INADA Naoki +Ionel Cristian Mărieș +Ionel Maries Cristian +Jakub Stasiak +Jakub Vysoky +James Cleveland +James Polley +Jan Pokorný +Jannis Leidel +jarondl +Jay Graves +Jeff Dairiki +Jim Garrison +John-Scott Atlakson +Jon Parise +Jon Wayne Parrott +Jonas Nockert +Jorge Niedbalski +Joseph Long +Josh Bronson +Josh Hansen +Josh Schneier +Jyrki Pulliainen +Kamal Bin Mustafa +Kelsey Hightower +Kenneth Belitzky +Kenneth Reitz +Kevin Burke +Kevin Frommelt +Kumar McMillan +Kyle Persohn +Laurent Bristiel +Leon Sasson +Lev Givon +Lincoln de Sousa +Ludovic Gasc +Luke Macken +Marc Abramowitz +Marc Tamlyn +Marcus Smith +Mark Kohler +Markus Hametner +Masklinn +Matej Stuchlik +Matt Good +Matt Maker +Matt Robenolt +Matthew Einhorn +Matthew Gilliard +Matthew Iversen +Matthew Trumbell +Maxime Rouyrre +Michael E. Karpeles +Michael Williamson +Miguel Araujo Perez +Mihir Singh +MinRK +Monty Taylor +Nick Stenning +Nowell Strite +Oliver Tonnhofer +Olivier Girardot +Ollie Rutherfurd +Oren Held +Oscar Benjamin +Patrick Dubroy +Patrick Jenkins +Patrick Lawson +patricktokeeffe +Paul Moore +Paul Nasrat +Paul Oswald +Paul van der Linden +Pawel Jasinski +Peter Waller +Phaneendra Chiruvella +Phil Freo +Phil Whelan +Philippe Ombredanne +Pierre-Yves Rofes +Piet Delport +Preston Holmes +Przemek Wrzos +Qiangning Hong +Rafael Caricio +Ralf Schmitt +Razzi Abuissa +Remi Rampin +Rene Dudfield +Richard Jones +RobberPhex +Robert Collins +Roey Berman +Roman Bogorodskiy +Romuald Brunet +Ronny Pfannschmidt +Rory McCann +Ross Brattain +schlamar +Sergey Vasilyev +Seth Woodworth +Simeon Visser +Simon Cross +Stavros Korokithakis +Stefan Scherfke +Steven Myint +Stéphane Klein +Takayuki SHIMIZUKAWA +Thomas Fenzl +Thomas Guettler +Thomas Johansson +Thomas Kluyver +Tim Harder +Tomer Chachamu +Toshio Kuratomi +Travis Swicegood +Valentin Haenel +Victor Stinner +Vinay Sajip +Vitaly Babiy +W. Trevor King +Wil Tan +Xavier Fernandez +Yoval P +Yu Jian +Zearin +Zhiping Deng \ No newline at end of file diff --git a/src/scancode/outdated.py b/src/scancode/outdated.py new file mode 100644 index 00000000000..07e523e2431 --- /dev/null +++ b/src/scancode/outdated.py @@ -0,0 +1,197 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. +# +# This code was in part derived from the pip library: +# Copyright (c) 2008-2014 The pip developers (see outdated.NOTICE file) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +from __future__ import absolute_import + +from collections import OrderedDict +import datetime +import json +import logging +from os import path +import sys + +from packaging import version as packaging_version +import requests +from requests.exceptions import ConnectionError +import yg.lockfile + +from scancode_config import scancode_cache_dir +from scancode_config import __version__ as scancode_version + + +SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" + + +logger = logging.getLogger(__name__) +# logging.basicConfig(stream=sys.stdout) +# logger.setLevel(logging.WARNING) + + +def total_seconds(td): + if hasattr(td, 'total_seconds'): + return td.total_seconds() + else: + val = td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6 + return val / 10 ** 6 + + +class VersionCheckState(object): + def __init__(self): + self.statefile_path = path.join( + scancode_cache_dir, 'scancode-version-check.json') + self.lockfile_path = self.statefile_path + '.lockfile' + # Load the existing state + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile) + except (IOError, ValueError, KeyError): + self.state = {} + + def save(self, latest_version, current_time): + # Attempt to write out our version check file + with yg.lockfile.FileLock(self.lockfile_path, timeout=10): + state = { + 'last_check': current_time.strftime(SELFCHECK_DATE_FMT), + 'latest_version': latest_version, + } + with open(self.statefile_path, 'w') as statefile: + json.dump(state, statefile, sort_keys=True, + separators=(',', ':')) + + +def check_scancode_version(installed_version=scancode_version, + new_version_url='https://pypi.org/pypi/scancode-toolkit/json', + force=False): + """ + Check for an updated version of scancode-toolkit. Return a message to + display if outdated or None. Limit the frequency of checks to once per week. + State is stored in the scancode_cache_dir. If `force` is True, redo a PyPI + remote check. + """ + installed_version = packaging_version.parse(installed_version) + latest_version = None + msg = None + + try: + state = VersionCheckState() + + current_time = datetime.datetime.utcnow() + # Determine if we need to refresh the state + if ('last_check' in state.state and 'latest_version' in state.state): + last_check = datetime.datetime.strptime( + state.state['last_check'], + SELFCHECK_DATE_FMT + ) + seconds_since_last_check = total_seconds(current_time - last_check) + one_week = 7 * 24 * 60 * 60 + if seconds_since_last_check < one_week: + latest_version = state.state['latest_version'] + + if force: + latest_version = None + + # Refresh the version if we need to or just see if we need to warn + if latest_version is None: + try: + latest_version = get_latest_version(new_version_url) + state.save(latest_version, current_time) + except Exception: + # save an empty version to avoid checking more than once a week + state.save(None, current_time) + raise + + latest_version = packaging_version.parse(latest_version) + + outdated_msg = ('WARNING: ' + 'You are using ScanCode Toolkit version %s, however the newer ' + 'version %s is available.\nYou should download and install the ' + 'latest version of ScanCode with bug and security fixes and the ' + 'latest license detection data for accurate scanning.\n' + 'Visit https://github.com/nexB/scancode-toolkit/releases for details.' + % (installed_version, latest_version) + ) + + # Determine if our latest_version is older + if (installed_version < latest_version + and installed_version.base_version != latest_version.base_version): + return outdated_msg + + except Exception: + msg = 'There was an error while checking for the latest version of ScanCode' + logger.debug(msg, exc_info=True) + + +def get_latest_version(new_version_url='https://pypi.org/pypi/scancode-toolkit/json'): + """ + Fetch `new_version_url` and return the latest version of scancode as a + string. + """ + requests_args = dict( + timeout=10, + verify=True, + headers={'Accept': 'application/json'}, + ) + try: + response = requests.get(new_version_url, **requests_args) + except (ConnectionError) as e: + logger.debug('get_latest_version: Download failed for %(url)r' % locals()) + raise + + status = response.status_code + if status != 200: + msg = 'get_latest_version: Download failed for %(url)r with %(status)r' % locals() + logger.debug(msg) + raise Exception(msg) + + # The check is done using python.org PyPI API + payload = response.json(object_pairs_hook=OrderedDict) + releases = [ + r for r in payload['releases'] if not packaging_version.parse(r).is_prerelease] + releases = sorted(releases, key=packaging_version.parse) + latest_version = releases[-1] + + return latest_version diff --git a/tests/scancode/test_outdated.py b/tests/scancode/test_outdated.py new file mode 100644 index 00000000000..38a35095e09 --- /dev/null +++ b/tests/scancode/test_outdated.py @@ -0,0 +1,151 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division +from __future__ import unicode_literals + + + +import pytest + +from commoncode.system import py3 +from scancode import outdated + + +pytestmark = pytest.mark.skipif(not py3, reason='Mock is not available as a builtin on py2') + + +def test_get_latest_version(): + from unittest import mock + pypi_mock_releases = { + 'releases': { + '2.0.0': [], + '2.0.0rc3': [], + '2.0.1': [], + '2.1.0': [], + '2.2.1': [], + '2.9.0b1': [], + '2.9.1': [], + '3.0.2': [], + '2.9.2': [], + '2.9.3': [], + '2.9.4': [], + '3.0.0': [], + } + } + def jget(*args, **kwargs): + return pypi_mock_releases + + with mock.patch('requests.get') as mock_get: + mock_get.return_value = mock.Mock( + json=jget, + status_code=200 + ) + result = outdated.get_latest_version() + assert '3.0.2' == result + + +def test_get_latest_version_fails_on_http_error(): + from unittest import mock + with mock.patch('requests.get') as mock_get: + mock_get.return_value = mock.Mock(status_code=400) + with pytest.raises(Exception): + outdated.get_latest_version() + + +def test_get_latest_version_ignore_rc_versions(): + from unittest import mock + pypi_mock_releases = { + 'releases': { + '2.0.0': [], + '2.0.0rc3': [], + '2.0.1': [], + '2.1.0': [], + '2.2.1': [], + '2.9.0rc3': [], + } + } + def jget(*args, **kwargs): + return pypi_mock_releases + + with mock.patch('requests.get') as mock_get: + mock_get.return_value = mock.Mock( + json=jget, + status_code=200 + ) + result = outdated.get_latest_version() + assert '2.2.1' == result + + +def test_check_scancode_version(): + from unittest import mock + pypi_mock_releases = { + 'releases': { + '2.0.0': [], + '2.0.0rc3': [], + '2.0.1': [], + '2.1.0': [], + '3.4.1': [], + '3.9.0rc3': [], + } + } + def jget(*args, **kwargs): + return pypi_mock_releases + + with mock.patch('requests.get') as mock_get: + mock_get.return_value = mock.Mock( + json=jget, + status_code=200 + ) + expected1 = 'You are using ScanCode Toolkit version' + expected2 = 'however the newer version 3.4.1 is available' + result = outdated.check_scancode_version(force=True) + assert expected1 in result + assert expected2 in result + + +def test_check_scancode_version_no_new_version(): + from unittest import mock + pypi_mock_releases = { + 'releases': { + '2.0.0': [], + '2.0.0rc3': [], + '2.0.1': [], + '2.1.0': [], + '3.0.1': [], + '3.9.0rc3': [], + } + } + def jget(*args, **kwargs): + return pypi_mock_releases + + with mock.patch('requests.get') as mock_get: + mock_get.return_value = mock.Mock( + json=jget, + status_code=200 + ) + result = outdated.check_scancode_version(force=True) + assert not result