From 560ccb72b96297395392d0117a52c62a9ef39030 Mon Sep 17 00:00:00 2001 From: alexzurbonsen Date: Thu, 5 Dec 2024 10:48:56 +0100 Subject: [PATCH] Fix failing --no-check-version cli option Currently, if the --no-check-version option is used, the scan fails because an unbound variable is passed to the run_scan function. Fix this by assigning None in that case. Add a test that passes with this change and fails without it. Signed-off-by: alexzurbonsen --- src/scancode/cli.py | 894 ++++++++++++++++++++++--------------- tests/scancode/test_cli.py | 6 + 2 files changed, 543 insertions(+), 357 deletions(-) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 1e189c89641..d3f12bebe38 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -36,6 +36,7 @@ class WindowsError(Exception): pass + import click # NOQA from commoncode import cliutils @@ -87,8 +88,8 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) - and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + echo_stderr = partial(click.secho, err=True) @@ -103,32 +104,42 @@ def print_examples(ctx, param, value): def print_version(ctx, param, value): if not value or ctx.resilient_parsing: return - click.echo(f'ScanCode version: {scancode_config.__version__}') - click.echo(f'ScanCode Output Format version: {scancode_config.__output_format_version__}') - click.echo(f'SPDX License list version: {scancode_config.spdx_license_list_version}') + click.echo(f"ScanCode version: {scancode_config.__version__}") + click.echo( + f"ScanCode Output Format version: {scancode_config.__output_format_version__}" + ) + click.echo( + f"SPDX License list version: {scancode_config.spdx_license_list_version}" + ) ctx.exit() class ScancodeCommand(GroupedHelpCommand): - short_usage_help = ''' -Try the 'scancode --help' option for help on options and arguments.''' + short_usage_help = """ +Try the 'scancode --help' option for help on options and arguments.""" try: # IMPORTANT: this discovers, loads and validates all available plugins plugin_classes, plugin_options = PluginManager.load_plugins() if TRACE: - logger_debug('plugin_options:') + logger_debug("plugin_options:") for clio in plugin_options: - logger_debug(' ', clio) - logger_debug('name:', clio.name, 'default:', clio.default) + logger_debug(" ", clio) + logger_debug("name:", clio.name, "default:", clio.default) except ImportError as e: - echo_stderr('========================================================================') - echo_stderr('ERROR: Unable to import ScanCode plugins.'.upper()) - echo_stderr('Check your installation configuration (setup.py) or re-install/re-configure ScanCode.') - echo_stderr('The following plugin(s) are referenced and cannot be loaded/imported:') - echo_stderr(str(e), color='red') - echo_stderr('========================================================================') + echo_stderr( + "========================================================================" + ) + echo_stderr("ERROR: Unable to import ScanCode plugins.".upper()) + echo_stderr( + "Check your installation configuration (setup.py) or re-install/re-configure ScanCode." + ) + echo_stderr("The following plugin(s) are referenced and cannot be loaded/imported:") + echo_stderr(str(e), color="red") + echo_stderr( + "========================================================================" + ) raise e @@ -136,25 +147,32 @@ def print_plugins(ctx, param, value): if not value or ctx.resilient_parsing: return for plugin_cls in sorted(plugin_classes, key=lambda pc: (pc.stage, pc.name)): - click.echo('--------------------------------------------') - click.echo('Plugin: scancode_{self.stage}:{self.name}'.format(self=plugin_cls), nl=False) - click.echo(' class: {self.__module__}:{self.__name__}'.format(self=plugin_cls)) - codebase_attributes = ', '.join(plugin_cls.codebase_attributes) - click.echo(' codebase_attributes: {}'.format(codebase_attributes)) - resource_attributes = ', '.join(plugin_cls.resource_attributes) - click.echo(' resource_attributes: {}'.format(resource_attributes)) - click.echo(' sort_order: {self.sort_order}'.format(self=plugin_cls)) - required_plugins = ', '.join(plugin_cls.required_plugins) - click.echo(' required_plugins: {}'.format(required_plugins)) - click.echo(' options:') + click.echo("--------------------------------------------") + click.echo( + "Plugin: scancode_{self.stage}:{self.name}".format(self=plugin_cls), + nl=False, + ) + click.echo(" class: {self.__module__}:{self.__name__}".format(self=plugin_cls)) + codebase_attributes = ", ".join(plugin_cls.codebase_attributes) + click.echo(" codebase_attributes: {}".format(codebase_attributes)) + resource_attributes = ", ".join(plugin_cls.resource_attributes) + click.echo(" resource_attributes: {}".format(resource_attributes)) + click.echo(" sort_order: {self.sort_order}".format(self=plugin_cls)) + required_plugins = ", ".join(plugin_cls.required_plugins) + click.echo(" required_plugins: {}".format(required_plugins)) + click.echo(" options:") for option in plugin_cls.options: name = option.name - opts = ', '.join(option.opts) + opts = ", ".join(option.opts) help_group = option.help_group help_txt = option.help # noqa - click.echo(' help_group: {help_group!s}, name: {name!s}: {opts}\n help: {help_txt!s}'.format(**locals())) - click.echo(' doc: {self.__doc__}'.format(self=plugin_cls)) - click.echo('') + click.echo( + " help_group: {help_group!s}, name: {name!s}: {opts}\n help: {help_txt!s}".format( + **locals() + ) + ) + click.echo(" doc: {self.__doc__}".format(self=plugin_cls)) + click.echo("") ctx.exit() @@ -162,11 +180,11 @@ def print_options(ctx, param, value): if not value or ctx.resilient_parsing: return values = ctx.params - click.echo('Options:') + click.echo("Options:") for name, val in sorted(values.items()): - click.echo(' {name}: {val}'.format(**locals())) - click.echo('') + click.echo(" {name}: {val}".format(**locals())) + click.echo("") ctx.exit() @@ -183,8 +201,13 @@ def validate_input_path(ctx, param, value): options = ctx.params from_json = options.get("--from-json", False) for inp in value: - if not (is_file(location=inp, follow_symlinks=True) or is_dir(location=inp, follow_symlinks=True)): - raise click.BadParameter(f"input: {inp!r} is not a regular file or a directory") + if not ( + is_file(location=inp, follow_symlinks=True) + or is_dir(location=inp, follow_symlinks=True) + ): + raise click.BadParameter( + f"input: {inp!r} is not a regular file or a directory" + ) if not is_readable(location=inp): raise click.BadParameter(f"input: {inp!r} is not readable") @@ -193,186 +216,258 @@ def validate_input_path(ctx, param, value): # extra JSON validation raise click.BadParameter(f"JSON input: {inp!r} is not a file") if not inp.lower().endswith(".json"): - raise click.BadParameter(f"JSON input: {inp!r} is not a JSON file with a .json extension") + raise click.BadParameter( + f"JSON input: {inp!r} is not a JSON file with a .json extension" + ) with open(inp) as js: start = js.read(100).strip() if not start.startswith("{"): - raise click.BadParameter(f"JSON input: {inp!r} is not a well formed JSON file") + raise click.BadParameter( + f"JSON input: {inp!r} is not a well formed JSON file" + ) return value -@click.command(name='scancode', +@click.command( + name="scancode", epilog=epilog_text, cls=ScancodeCommand, - plugin_options=plugin_options) - + plugin_options=plugin_options, +) @click.pass_context - -@click.argument('input', - metavar=' ...', nargs=-1, +@click.argument( + "input", + metavar=" ...", + nargs=-1, callback=validate_input_path, - type=click.Path(exists=True, readable=True, path_type=str)) - -@click.option('--strip-root', + type=click.Path(exists=True, readable=True, path_type=str), +) +@click.option( + "--strip-root", is_flag=True, - conflicting_options=['full_root'], - help='Strip the root directory segment of all paths. The default is to ' - 'always include the last directory segment of the scanned path such ' - 'that all paths have a common root directory.', - help_group=cliutils.OUTPUT_CONTROL_GROUP, cls=PluggableCommandLineOption) - -@click.option('--full-root', + conflicting_options=["full_root"], + help="Strip the root directory segment of all paths. The default is to " + "always include the last directory segment of the scanned path such " + "that all paths have a common root directory.", + help_group=cliutils.OUTPUT_CONTROL_GROUP, + cls=PluggableCommandLineOption, +) +@click.option( + "--full-root", is_flag=True, - conflicting_options=['strip_root'], - help='Report full, absolute paths.', - help_group=cliutils.OUTPUT_CONTROL_GROUP, cls=PluggableCommandLineOption) - -@click.option('-n', '--processes', + conflicting_options=["strip_root"], + help="Report full, absolute paths.", + help_group=cliutils.OUTPUT_CONTROL_GROUP, + cls=PluggableCommandLineOption, +) +@click.option( + "-n", + "--processes", type=int, default=1, - metavar='INT', - help='Set the number of parallel processes to use. ' - 'Disable parallel processing if 0. Also disable threading if -1. [default: 1]', - help_group=cliutils.CORE_GROUP, sort_order=10, cls=PluggableCommandLineOption) - -@click.option('--timeout', + metavar="INT", + help="Set the number of parallel processes to use. " + "Disable parallel processing if 0. Also disable threading if -1. [default: 1]", + help_group=cliutils.CORE_GROUP, + sort_order=10, + cls=PluggableCommandLineOption, +) +@click.option( + "--timeout", type=float, default=DEFAULT_TIMEOUT, - metavar='', - help='Stop an unfinished file scan after a timeout in seconds. ' - f'[default: {DEFAULT_TIMEOUT} seconds]', - help_group=cliutils.CORE_GROUP, sort_order=10, cls=PluggableCommandLineOption) - -@click.option('-q', '--quiet', + metavar="", + help="Stop an unfinished file scan after a timeout in seconds. " + f"[default: {DEFAULT_TIMEOUT} seconds]", + help_group=cliutils.CORE_GROUP, + sort_order=10, + cls=PluggableCommandLineOption, +) +@click.option( + "-q", + "--quiet", is_flag=True, - conflicting_options=['verbose'], - help='Do not print summary or progress.', - help_group=cliutils.CORE_GROUP, sort_order=20, cls=PluggableCommandLineOption) - -@click.option('-v', '--verbose', + conflicting_options=["verbose"], + help="Do not print summary or progress.", + help_group=cliutils.CORE_GROUP, + sort_order=20, + cls=PluggableCommandLineOption, +) +@click.option( + "-v", + "--verbose", is_flag=True, - conflicting_options=['quiet'], - help='Print progress as file-by-file path instead of a progress bar. ' - 'Print verbose scan counters.', - help_group=cliutils.CORE_GROUP, sort_order=20, cls=PluggableCommandLineOption) - -@click.option('--from-json', + conflicting_options=["quiet"], + help="Print progress as file-by-file path instead of a progress bar. " + "Print verbose scan counters.", + help_group=cliutils.CORE_GROUP, + sort_order=20, + cls=PluggableCommandLineOption, +) +@click.option( + "--from-json", is_flag=True, - help='Load codebase from one or more JSON scan file(s).', - help_group=cliutils.CORE_GROUP, sort_order=25, cls=PluggableCommandLineOption) - -@click.option('--timing', + help="Load codebase from one or more JSON scan file(s).", + help_group=cliutils.CORE_GROUP, + sort_order=25, + cls=PluggableCommandLineOption, +) +@click.option( + "--timing", is_flag=True, hidden=True, - help='Collect scan timing for each scan/scanned file.', - help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption) - -@click.option('--max-in-memory', - type=int, default=10000, + help="Collect scan timing for each scan/scanned file.", + help_group=cliutils.CORE_GROUP, + sort_order=250, + cls=PluggableCommandLineOption, +) +@click.option( + "--max-in-memory", + type=int, + default=10000, show_default=True, - help='Maximum number of files and directories scan details kept in memory ' - 'during a scan. Additional files and directories scan details above this ' - 'number are cached on-disk rather than in memory. ' - 'Use 0 to use unlimited memory and disable on-disk caching. ' - 'Use -1 to use only on-disk caching.', - help_group=cliutils.CORE_GROUP, sort_order=300, cls=PluggableCommandLineOption) - -@click.option('--max-depth', + help="Maximum number of files and directories scan details kept in memory " + "during a scan. Additional files and directories scan details above this " + "number are cached on-disk rather than in memory. " + "Use 0 to use unlimited memory and disable on-disk caching. " + "Use -1 to use only on-disk caching.", + help_group=cliutils.CORE_GROUP, + sort_order=300, + cls=PluggableCommandLineOption, +) +@click.option( + "--max-depth", type=int, default=0, show_default=False, callback=validate_depth, - help='Maximum nesting depth of subdirectories to scan. ' - 'Descend at most INTEGER levels of directories below and including ' - 'the starting directory. Use 0 for no scan depth limit.', - help_group=cliutils.CORE_GROUP, sort_order=301, cls=PluggableCommandLineOption) - -@click.help_option('-h', '--help', - help_group=cliutils.DOC_GROUP, sort_order=10, cls=PluggableCommandLineOption) - -@click.option('-A', '--about', + help="Maximum nesting depth of subdirectories to scan. " + "Descend at most INTEGER levels of directories below and including " + "the starting directory. Use 0 for no scan depth limit.", + help_group=cliutils.CORE_GROUP, + sort_order=301, + cls=PluggableCommandLineOption, +) +@click.help_option( + "-h", + "--help", + help_group=cliutils.DOC_GROUP, + sort_order=10, + cls=PluggableCommandLineOption, +) +@click.option( + "-A", + "--about", is_flag=True, is_eager=True, expose_value=False, callback=print_about, - help='Show information about ScanCode and licensing and exit.', - help_group=cliutils.DOC_GROUP, sort_order=20, cls=PluggableCommandLineOption) - -@click.option('-V', '--version', + help="Show information about ScanCode and licensing and exit.", + help_group=cliutils.DOC_GROUP, + sort_order=20, + cls=PluggableCommandLineOption, +) +@click.option( + "-V", + "--version", is_flag=True, is_eager=True, expose_value=False, callback=print_version, - help='Show the version and exit.', - help_group=cliutils.DOC_GROUP, sort_order=20, cls=PluggableCommandLineOption) - -@click.option('--examples', + help="Show the version and exit.", + help_group=cliutils.DOC_GROUP, + sort_order=20, + cls=PluggableCommandLineOption, +) +@click.option( + "--examples", is_flag=True, is_eager=True, expose_value=False, callback=print_examples, - help=('Show command examples and exit.'), - help_group=cliutils.DOC_GROUP, sort_order=50, cls=PluggableCommandLineOption) - -@click.option('--plugins', + help=("Show command examples and exit."), + help_group=cliutils.DOC_GROUP, + sort_order=50, + cls=PluggableCommandLineOption, +) +@click.option( + "--plugins", is_flag=True, is_eager=True, expose_value=False, callback=print_plugins, - help='Show the list of available ScanCode plugins and exit.', - help_group=cliutils.DOC_GROUP, cls=PluggableCommandLineOption) - -@click.option('--test-mode', + help="Show the list of available ScanCode plugins and exit.", + help_group=cliutils.DOC_GROUP, + cls=PluggableCommandLineOption, +) +@click.option( + "--test-mode", is_flag=True, default=False, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, help='Run ScanCode in a special "test mode". Only for testing.', - help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) - -@click.option('--test-slow-mode', + help_group=cliutils.MISC_GROUP, + sort_order=1000, + cls=PluggableCommandLineOption, +) +@click.option( + "--test-slow-mode", is_flag=True, default=False, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, help='Run ScanCode in a special "test slow mode" to ensure that --email ' - 'scan needs at least one second to complete. Only for testing.', - help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) - -@click.option('--test-error-mode', + "scan needs at least one second to complete. Only for testing.", + help_group=cliutils.MISC_GROUP, + sort_order=1000, + cls=PluggableCommandLineOption, +) +@click.option( + "--test-error-mode", is_flag=True, default=False, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, help='Run ScanCode in a special "test error mode" to trigger errors with ' - 'the --email scan. Only for testing.', - help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) - -@click.option('--print-options', + "the --email scan. Only for testing.", + help_group=cliutils.MISC_GROUP, + sort_order=1000, + cls=PluggableCommandLineOption, +) +@click.option( + "--print-options", is_flag=True, expose_value=False, callback=print_options, - help='Show the list of selected options and exit.', - help_group=cliutils.DOC_GROUP, cls=PluggableCommandLineOption) - -@click.option('--keep-temp-files', + help="Show the list of selected options and exit.", + help_group=cliutils.DOC_GROUP, + cls=PluggableCommandLineOption, +) +@click.option( + "--keep-temp-files", is_flag=True, default=False, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, - help='Keep temporary files and show the directory where temporary files ' - 'are stored. (By default temporary files are deleted when a scan is ' - 'completed.)', - help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) - + help="Keep temporary files and show the directory where temporary files " + "are stored. (By default temporary files are deleted when a scan is " + "completed.)", + help_group=cliutils.MISC_GROUP, + sort_order=1000, + cls=PluggableCommandLineOption, +) @click.option( "--check-version/--no-check-version", help="Whether to check for new versions. Defaults to true.", default=True, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, - help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) + help_group=cliutils.MISC_GROUP, + sort_order=1000, + cls=PluggableCommandLineOption, +) def scancode( ctx, input, # NOQA @@ -469,9 +564,9 @@ def scancode( try: # Validate CLI UI options dependencies and other CLI-specific inits if TRACE_DEEP: - logger_debug('scancode: ctx.params:') + logger_debug("scancode: ctx.params:") for co in sorted(ctx.params.items()): - logger_debug(' scancode: ctx.params:', co) + logger_debug(" scancode: ctx.params:", co) cliutils.validate_option_dependencies(ctx) pretty_params = get_pretty_params(ctx, generic_paths=test_mode) @@ -479,7 +574,10 @@ def scancode( # Check for updates if check_version: from scancode.outdated import check_scancode_version + outdated = check_scancode_version() + else: + outdated = None # run proper success, _results = run_scan( @@ -504,12 +602,12 @@ def scancode( echo_func=echo_func, outdated=outdated, *args, - **kwargs + **kwargs, ) # echo outdated message if newer version is available if not quiet and outdated: - echo_stderr(outdated, fg='yellow') + echo_stderr(outdated, fg="yellow") except click.UsageError as e: # this will exit @@ -547,7 +645,7 @@ def run_scan( plugin_options=plugin_options, outdated=None, *args, - **kwargs + **kwargs, ): """ Run a scan on `input` path (or a list of input paths) and return a tuple of @@ -557,7 +655,9 @@ def run_scan( error. See scancode() for arguments details. """ - assert not (return_results is True and return_codebase is True), 'Only one of return_results and return_codebase can be True' + assert not ( + return_results is True and return_codebase is True + ), "Only one of return_results and return_codebase can be True" if return_codebase and max_in_memory > 0: # We're keeping temp files otherwise the codebase is gone @@ -573,7 +673,7 @@ def echo_func(*_args, **_kwargs): pass if not input: - msg = 'At least one input path is required.' + msg = "At least one input path is required." raise ScancodeError(msg) if not isinstance(input, (list, tuple)): @@ -598,8 +698,8 @@ def echo_func(*_args, **_kwargs): if any(os.path.isabs(p) for p in input): msg = ( - 'Invalid inputs: all input paths must be relative when ' - 'using multiple inputs.' + "Invalid inputs: all input paths must be relative when " + "using multiple inputs." ) raise ScancodeError(msg) @@ -610,23 +710,23 @@ def echo_func(*_args, **_kwargs): if not common_prefix: # we have no common prefix, but all relative. therefore the # parent/root is the current ddirectory - common_prefix = str('.') + common_prefix = str(".") elif not os.path.isdir(common_prefix): msg = ( - 'Invalid inputs: all input paths must share a ' - 'common single parent directory.' + "Invalid inputs: all input paths must share a " + "common single parent directory." ) raise ScancodeError(msg) # and we craft a list of synthetic --include path pattern options from # the input list of paths - included_paths = [as_posixpath(path).rstrip('/') for path in input] + included_paths = [as_posixpath(path).rstrip("/") for path in input] # FIXME: this is a hack as this "include" is from an external plugin!!! - include = list(requested_options.get('include', []) or []) + include = list(requested_options.get("include", []) or []) include.extend(included_paths) - requested_options['include'] = include + requested_options["include"] = include # ... and use the common prefix as our new input input = common_prefix # NOQA @@ -659,11 +759,13 @@ def echo_func(*_args, **_kwargs): if not quiet: if not processes: - echo_func('Disabling multi-processing for debugging.', fg='yellow') + echo_func("Disabling multi-processing for debugging.", fg="yellow") elif processes == -1: - echo_func('Disabling multi-processing ' - 'and multi-threading for debugging.', fg='yellow') + echo_func( + "Disabling multi-processing " "and multi-threading for debugging.", + fg="yellow", + ) try: ######################################################################## @@ -685,7 +787,7 @@ def echo_func(*_args, **_kwargs): try: is_enabled = plugin.is_enabled(**requested_options) except TypeError as te: - if not 'takes exactly' in str(te): + if not "takes exactly" in str(te): raise te if is_enabled: stage_plugins.append(plugin) @@ -693,8 +795,8 @@ def echo_func(*_args, **_kwargs): else: non_enabled_plugins_by_qname[qname] = plugin except: - msg = 'Failed to load plugin: %(qname)s:' % locals() - raise ScancodeError(msg + '\n' + traceback.format_exc()) + msg = "Failed to load plugin: %(qname)s:" % locals() + raise ScancodeError(msg + "\n" + traceback.format_exc()) # NOTE: these are list of plugin instances, not classes! pre_scan_plugins = enabled_plugins_by_stage[pre_scan.stage] @@ -704,13 +806,17 @@ def echo_func(*_args, **_kwargs): output_plugins = enabled_plugins_by_stage[output.stage] if from_json and scanner_plugins: - msg = ('ERROR: Data loaded from JSON: no file scan options can be ' - 'selected when using only scan data.') + msg = ( + "ERROR: Data loaded from JSON: no file scan options can be " + "selected when using only scan data." + ) raise ScancodeCliUsageError(msg) if not output_plugins and not (return_results or return_codebase): - msg = ('ERROR: Missing output option(s): at least one output ' - 'option is required to save scan results.') + msg = ( + "ERROR: Missing output option(s): at least one output " + "option is required to save scan results." + ) raise ScancodeCliUsageError(msg) ######################################################################## @@ -720,8 +826,12 @@ def echo_func(*_args, **_kwargs): requestors_by_missing_qname = defaultdict(set) if TRACE_DEEP: - logger_debug('scancode: all_enabled_plugins_by_qname:', all_enabled_plugins_by_qname) - logger_debug('scancode: non_enabled_plugins_by_qname:', non_enabled_plugins_by_qname) + logger_debug( + "scancode: all_enabled_plugins_by_qname:", all_enabled_plugins_by_qname + ) + logger_debug( + "scancode: non_enabled_plugins_by_qname:", non_enabled_plugins_by_qname + ) for qname, enabled_plugin in all_enabled_plugins_by_qname.items(): for required_qname in enabled_plugin.required_plugins or []: @@ -729,25 +839,31 @@ def echo_func(*_args, **_kwargs): # there is nothing to do since we have it already as enabled pass elif required_qname in non_enabled_plugins_by_qname: - plugins_to_setup.append(non_enabled_plugins_by_qname[required_qname]) + plugins_to_setup.append( + non_enabled_plugins_by_qname[required_qname] + ) else: # we have a required but not loaded plugin requestors_by_missing_qname[required_qname].add(qname) if requestors_by_missing_qname: - msg = 'Some required plugins are missing from available plugins:\n' + msg = "Some required plugins are missing from available plugins:\n" for qn, requestors in requestors_by_missing_qname.items(): - rqs = ', '.join(sorted(requestors)) - msg += ' Plugin: {qn} is required by plugins: {rqs}.\n'.format(**locals()) + rqs = ", ".join(sorted(requestors)) + msg += " Plugin: {qn} is required by plugins: {rqs}.\n".format( + **locals() + ) raise ScancodeError(msg) if TRACE_DEEP: - logger_debug('scancode: plugins_to_setup: from required:', plugins_to_setup) + logger_debug("scancode: plugins_to_setup: from required:", plugins_to_setup) plugins_to_setup.extend(all_enabled_plugins_by_qname.values()) if TRACE_DEEP: - logger_debug('scancode: plugins_to_setup: includng enabled:', plugins_to_setup) + logger_debug( + "scancode: plugins_to_setup: includng enabled:", plugins_to_setup + ) ######################################################################## # Setup enabled and required plugins @@ -757,7 +873,7 @@ def echo_func(*_args, **_kwargs): plugins_setup_start = time() if not quiet and not verbose: - echo_func('Setup plugins...', fg='green') + echo_func("Setup plugins...", fg="green") # TODO: add progress indicator for plugin in plugins_to_setup: @@ -765,18 +881,17 @@ def echo_func(*_args, **_kwargs): stage = plugin.stage name = plugin.name if verbose: - echo_func(' Setup plugin: %(stage)s:%(name)s...' % locals(), - fg='green') + echo_func(" Setup plugin: %(stage)s:%(name)s..." % locals(), fg="green") try: plugin.setup(**requested_options) except: - msg = 'ERROR: failed to setup plugin: %(stage)s:%(name)s:' % locals() - raise ScancodeError(msg + '\n' + traceback.format_exc()) + msg = "ERROR: failed to setup plugin: %(stage)s:%(name)s:" % locals() + raise ScancodeError(msg + "\n" + traceback.format_exc()) - timing_key = 'setup_%(stage)s:%(name)s' % locals() + timing_key = "setup_%(stage)s:%(name)s" % locals() setup_timings[timing_key] = time() - plugin_setup_start - setup_timings['setup'] = time() - plugins_setup_start + setup_timings["setup"] = time() - plugins_setup_start ######################################################################## # Collect Resource attributes requested for this scan @@ -786,19 +901,29 @@ def echo_func(*_args, **_kwargs): # mapping of {"plugin stage:name": [list of attribute keys]} # also available as a kwarg entry for plugin - requested_options['resource_attributes_by_plugin'] = resource_attributes_by_plugin = {} + requested_options["resource_attributes_by_plugin"] = ( + resource_attributes_by_plugin + ) = {} for stage, stage_plugins in enabled_plugins_by_stage.items(): for plugin in stage_plugins: name = plugin.name try: sortable_resource_attributes.append( - (plugin.sort_order, name, plugin.resource_attributes,) + ( + plugin.sort_order, + name, + plugin.resource_attributes, + ) + ) + resource_attributes_by_plugin[plugin.qname()] = ( + plugin.resource_attributes.keys() ) - resource_attributes_by_plugin[plugin.qname()] = plugin.resource_attributes.keys() except: - msg = ('ERROR: failed to collect resource_attributes for plugin: ' - '%(stage)s:%(name)s:' % locals()) - raise ScancodeError(msg + '\n' + traceback.format_exc()) + msg = ( + "ERROR: failed to collect resource_attributes for plugin: " + "%(stage)s:%(name)s:" % locals() + ) + raise ScancodeError(msg + "\n" + traceback.format_exc()) resource_attributes = {} for _, name, attribs in sorted(sortable_resource_attributes): @@ -811,7 +936,7 @@ def echo_func(*_args, **_kwargs): attrib.counter = order if TRACE_DEEP: - logger_debug('scancode:resource_attributes') + logger_debug("scancode:resource_attributes") for a in resource_attributes.items(): logger_debug(a) @@ -822,19 +947,29 @@ def echo_func(*_args, **_kwargs): # mapping of {"plugin stage:name": [list of attribute keys]} # also available as a kwarg entry for plugin - requested_options['codebase_attributes_by_plugin'] = codebase_attributes_by_plugin = {} + requested_options["codebase_attributes_by_plugin"] = ( + codebase_attributes_by_plugin + ) = {} for stage, stage_plugins in enabled_plugins_by_stage.items(): for plugin in stage_plugins: name = plugin.name try: sortable_codebase_attributes.append( - (plugin.sort_order, name, plugin.codebase_attributes,) + ( + plugin.sort_order, + name, + plugin.codebase_attributes, + ) + ) + codebase_attributes_by_plugin[plugin.qname()] = ( + plugin.codebase_attributes.keys() ) - codebase_attributes_by_plugin[plugin.qname()] = plugin.codebase_attributes.keys() except: - msg = ('ERROR: failed to collect codebase_attributes for plugin: ' - '%(stage)s:%(name)s:' % locals()) - raise ScancodeError(msg + '\n' + traceback.format_exc()) + msg = ( + "ERROR: failed to collect codebase_attributes for plugin: " + "%(stage)s:%(name)s:" % locals() + ) + raise ScancodeError(msg + "\n" + traceback.format_exc()) codebase_attributes = {} for _, name, attribs in sorted(sortable_codebase_attributes): @@ -847,7 +982,7 @@ def echo_func(*_args, **_kwargs): attrib.counter = order if TRACE_DEEP: - logger_debug('scancode:codebase_attributes') + logger_debug("scancode:codebase_attributes") for a in codebase_attributes.items(): logger_debug(a) @@ -858,14 +993,16 @@ def echo_func(*_args, **_kwargs): inventory_start = time() if not quiet: - echo_func('Collect file inventory...', fg='green') + echo_func("Collect file inventory...", fg="green") if from_json: codebase_class = VirtualCodebase - codebase_load_error_msg = 'ERROR: failed to load codebase from scan file at: %(input)r' + codebase_load_error_msg = ( + "ERROR: failed to load codebase from scan file at: %(input)r" + ) else: codebase_class = Codebase - codebase_load_error_msg = 'ERROR: failed to collect codebase at: %(input)r' + codebase_load_error_msg = "ERROR: failed to collect codebase at: %(input)r" # TODO: add progress indicator # Note: inventory timing collection is built in Codebase initialization @@ -881,41 +1018,47 @@ def echo_func(*_args, **_kwargs): max_depth=max_depth, ) except Exception as e: - if from_json and isinstance(e, (json.decoder.JSONDecodeError, UnicodeDecodeError)): - raise click.BadParameter(f"Input JSON scan file(s) is not valid JSON: {input!r} : {e!r}") + if from_json and isinstance( + e, (json.decoder.JSONDecodeError, UnicodeDecodeError) + ): + raise click.BadParameter( + f"Input JSON scan file(s) is not valid JSON: {input!r} : {e!r}" + ) else: - msg = f'Failed to process codebase at: {input!r}' - raise ScancodeError(msg + '\n' + traceback.format_exc()) + msg = f"Failed to process codebase at: {input!r}" + raise ScancodeError(msg + "\n" + traceback.format_exc()) # update headers cle = codebase.get_or_create_current_header() cle.start_timestamp = start_timestamp - cle.tool_name = 'scancode-toolkit' + cle.tool_name = "scancode-toolkit" cle.tool_version = scancode_config.__version__ cle.output_format_version = scancode_config.__output_format_version__ cle.notice = notice cle.options = pretty_params or {} # useful for debugging - cle.extra_data['system_environment'] = system_environment = {} + cle.extra_data["system_environment"] = system_environment = {} - system_environment['operating_system'] = commoncode.system.current_os - system_environment['cpu_architecture'] = commoncode.system.current_arch - system_environment['platform'] = platform.platform() - system_environment['platform_version'] = platform.version() - system_environment['python_version'] = sys.version + system_environment["operating_system"] = commoncode.system.current_os + system_environment["cpu_architecture"] = commoncode.system.current_arch + system_environment["platform"] = platform.platform() + system_environment["platform_version"] = platform.version() + system_environment["python_version"] = sys.version - cle.extra_data['spdx_license_list_version'] = scancode_config.spdx_license_list_version + cle.extra_data["spdx_license_list_version"] = ( + scancode_config.spdx_license_list_version + ) if outdated: - cle.extra_data['OUTDATED'] = outdated + cle.extra_data["OUTDATED"] = outdated # TODO: this is weird: may be the timings should NOT be stored on the # codebase, since they exist in abstract of it?? codebase.timings.update(setup_timings) - codebase.timings['inventory'] = time() - inventory_start + codebase.timings["inventory"] = time() - inventory_start files_count, dirs_count, size_count = codebase.compute_counts() - codebase.counters['initial:files_count'] = files_count - codebase.counters['initial:dirs_count'] = dirs_count - codebase.counters['initial:size_count'] = size_count + codebase.counters["initial:files_count"] = files_count + codebase.counters["initial:dirs_count"] = dirs_count + codebase.counters["initial:size_count"] = size_count ######################################################################## # Run prescans @@ -923,11 +1066,11 @@ def echo_func(*_args, **_kwargs): # TODO: add progress indicator pre_scan_success = run_codebase_plugins( - stage='pre-scan', + stage="pre-scan", plugins=pre_scan_plugins, codebase=codebase, - stage_msg='Run %(stage)ss...', - plugin_msg=' Run %(stage)s: %(name)s...', + stage_msg="Run %(stage)ss...", + plugin_msg=" Run %(stage)s: %(name)s...", quiet=quiet, verbose=verbose, kwargs=requested_options, @@ -940,7 +1083,7 @@ def echo_func(*_args, **_kwargs): ######################################################################## scan_success = run_scanners( - stage='scan', + stage="scan", plugins=scanner_plugins, codebase=codebase, processes=processes, @@ -959,11 +1102,11 @@ def echo_func(*_args, **_kwargs): # TODO: add progress indicator post_scan_success = run_codebase_plugins( - stage='post-scan', + stage="post-scan", plugins=post_scan_plugins, codebase=codebase, - stage_msg='Run %(stage)ss...', - plugin_msg=' Run %(stage)s: %(name)s...', + stage_msg="Run %(stage)ss...", + plugin_msg=" Run %(stage)s: %(name)s...", quiet=quiet, verbose=verbose, kwargs=requested_options, @@ -977,11 +1120,11 @@ def echo_func(*_args, **_kwargs): # TODO: add progress indicator output_filter_success = run_codebase_plugins( - stage='output-filter', + stage="output-filter", plugins=output_filter_plugins, codebase=codebase, - stage_msg='Apply %(stage)ss...', - plugin_msg=' Apply %(stage)s: %(name)s...', + stage_msg="Apply %(stage)ss...", + plugin_msg=" Apply %(stage)s: %(name)s...", quiet=quiet, verbose=verbose, kwargs=requested_options, @@ -996,9 +1139,9 @@ def echo_func(*_args, **_kwargs): counts = codebase.compute_counts(skip_root=strip_root, skip_filtered=True) files_count, dirs_count, size_count = counts - codebase.counters['final:files_count'] = files_count - codebase.counters['final:dirs_count'] = dirs_count - codebase.counters['final:size_count'] = size_count + codebase.counters["final:files_count"] = files_count + codebase.counters["final:dirs_count"] = dirs_count + codebase.counters["final:size_count"] = size_count cle.end_timestamp = time2tstamp() cle.duration = time() - processing_start @@ -1011,11 +1154,11 @@ def echo_func(*_args, **_kwargs): if output_plugins: # TODO: add progress indicator output_success = run_codebase_plugins( - stage='output', + stage="output", plugins=output_plugins, codebase=codebase, - stage_msg='Save scan results...', - plugin_msg=' Save scan results as: %(name)s...', + stage_msg="Save scan results...", + plugin_msg=" Save scan results as: %(name)s...", quiet=quiet, verbose=verbose, kwargs=requested_options, @@ -1026,12 +1169,12 @@ def echo_func(*_args, **_kwargs): ######################################################################## # Display summary ######################################################################## - codebase.timings['total'] = time() - processing_start + codebase.timings["total"] = time() - processing_start # TODO: compute summary for output plugins too?? if not quiet: - scan_names = ', '.join(p.name for p in scanner_plugins) - echo_func('Scanning done.', fg='green' if success else 'red') + scan_names = ", ".join(p.name for p in scanner_plugins) + echo_func("Scanning done.", fg="green" if success else "red") display_summary( codebase=codebase, scan_names=scan_names, @@ -1046,6 +1189,7 @@ def echo_func(*_args, **_kwargs): if return_results: # the structure is exactly the same as the JSON output from formattedcode.output_json import get_results + results = get_results(codebase, as_list=True, **requested_options) elif return_codebase: results = codebase @@ -1056,16 +1200,17 @@ def echo_func(*_args, **_kwargs): if keep_temp_files: if not quiet: msg = 'Keeping temporary files in: "{}".'.format(scancode_temp_dir) - echo_func(msg, fg='green' if success else 'red') + echo_func(msg, fg="green" if success else "red") else: if not quiet: - echo_func('Removing temporary files...', fg='green', nl=False) + echo_func("Removing temporary files...", fg="green", nl=False) from commoncode import fileutils + fileutils.delete(scancode_temp_dir) if not quiet: - echo_func('done.', fg='green') + echo_func("done.", fg="green") return success, results @@ -1074,8 +1219,8 @@ def run_codebase_plugins( stage, plugins, codebase, - stage_msg='', - plugin_msg='', + stage_msg="", + plugin_msg="", quiet=False, verbose=False, kwargs=None, @@ -1093,7 +1238,7 @@ def run_codebase_plugins( stage_start = time() if verbose and plugins: - echo_func(stage_msg % locals(), fg='green') + echo_func(stage_msg % locals(), fg="green") # Sort plugins by run_order, from low to high sorted_plugins = sorted(plugins, key=lambda x: x.run_order) @@ -1105,26 +1250,30 @@ def run_codebase_plugins( plugin_start = time() if verbose: - echo_func(plugin_msg % locals(), fg='green') + echo_func(plugin_msg % locals(), fg="green") try: if TRACE_DEEP: from pprint import pformat - logger_debug('run_codebase_plugins: kwargs passed to %(stage)s:%(name)s' % locals()) + + logger_debug( + "run_codebase_plugins: kwargs passed to %(stage)s:%(name)s" + % locals() + ) logger_debug(pformat(sorted(kwargs.items()))) logger_debug() plugin.process_codebase(codebase, **kwargs) except Exception as _e: - msg = 'ERROR: failed to run %(stage)s plugin: %(name)s:' % locals() - echo_func(msg, fg='red') + msg = "ERROR: failed to run %(stage)s plugin: %(name)s:" % locals() + echo_func(msg, fg="red") tb = traceback.format_exc() echo_func(tb) - codebase.errors.append(msg + '\n' + tb) + codebase.errors.append(msg + "\n" + tb) success = False - timing_key = '%(stage)s:%(name)s' % locals() + timing_key = "%(stage)s:%(name)s" % locals() codebase.timings[timing_key] = time() - plugin_start codebase.timings[stage] = time() - stage_start @@ -1166,32 +1315,45 @@ def run_scanners( func = plugin.get_scanner(**kwargs) scanners.append(Scanner(name=plugin.name, function=func)) - if TRACE_DEEP: logger_debug('run_scanners: scanners:', scanners) + if TRACE_DEEP: + logger_debug("run_scanners: scanners:", scanners) if not scanners: return True - scan_names = ', '.join(s.name for s in scanners) + scan_names = ", ".join(s.name for s in scanners) progress_manager = None if not quiet: - echo_func(f'Scan files for: {scan_names} with {processes} process(es)...') + echo_func(f"Scan files for: {scan_names} with {processes} process(es)...") item_show_func = partial(path_progress_message, verbose=verbose) - progress_manager = partial(progressmanager, + progress_manager = partial( + progressmanager, item_show_func=item_show_func, - verbose=verbose, file=sys.stderr) + verbose=verbose, + file=sys.stderr, + ) # TODO: add CLI option to bypass cache entirely? scan_success = scan_codebase( - codebase, scanners, processes, timeout, - with_timing=timing, progress_manager=progress_manager) + codebase, + scanners, + processes, + timeout, + with_timing=timing, + progress_manager=progress_manager, + ) # TODO: add progress indicator # run the process codebase of each scan plugin (most often a no-op) scan_process_codebase_success = run_codebase_plugins( - stage, plugins, codebase, - stage_msg='Filter %(stage)ss...', - plugin_msg=' Filter %(stage)s: %(name)s...', - quiet=quiet, verbose=verbose, kwargs=kwargs, + stage, + plugins, + codebase, + stage_msg="Filter %(stage)ss...", + plugin_msg=" Filter %(stage)s: %(name)s...", + quiet=quiet, + verbose=verbose, + kwargs=kwargs, ) scan_success = scan_success and scan_process_codebase_success @@ -1200,15 +1362,15 @@ def run_scanners( try: scanned_fc, scanned_dc, scanned_sc = codebase.compute_counts() except: - msg = 'ERROR: in run_scanners, failed to compute codebase counts:\n' + msg = "ERROR: in run_scanners, failed to compute codebase counts:\n" msg += traceback.format_exc() codebase.errors.append(msg) scan_success = False - codebase.counters[stage + ':scanners'] = scan_names - codebase.counters[stage + ':files_count'] = scanned_fc - codebase.counters[stage + ':dirs_count'] = scanned_dc - codebase.counters[stage + ':size_count'] = scanned_sc + codebase.counters[stage + ":scanners"] = scan_names + codebase.counters[stage + ":files_count"] = scanned_fc + codebase.counters[stage + ":dirs_count"] = scanned_dc + codebase.counters[stage + ":size_count"] = scanned_sc return scan_success @@ -1250,11 +1412,11 @@ def scan_codebase( scanners=scanners, timeout=timeout, with_timing=with_timing, - with_threading=use_threading + with_threading=use_threading, ) if TRACE: - logger_debug('scan_codebase: scanners:', ', '.join(s.name for s in scanners)) + logger_debug("scan_codebase: scanners:", ", ".join(s.name for s in scanners)) get_resource = codebase.get_resource @@ -1279,30 +1441,28 @@ def scan_codebase( if progress_manager: scans = progress_manager(scans) # hack to avoid using a context manager - if hasattr(scans, '__enter__'): + if hasattr(scans, "__enter__"): scans.__enter__() while True: try: - (location, - path, - scan_errors, - scan_time, - scan_result, - scan_timings) = next(scans) + (location, path, scan_errors, scan_time, scan_result, scan_timings) = ( + next(scans) + ) if TRACE_DEEP: logger_debug( - 'scan_codebase: location:', location, 'results:', scan_result) + "scan_codebase: location:", location, "results:", scan_result + ) resource = get_resource(path=path) if not resource: # this should never happen msg = ( - 'ERROR: Internal error in scan_codebase: Resource ' - f'at {path!r} is missing from codebase.\n' - f'Scan result not saved:\n{scan_result!r}.' + "ERROR: Internal error in scan_codebase: Resource " + f"at {path!r} is missing from codebase.\n" + f"Scan result not saved:\n{scan_result!r}." ) codebase.errors.append(msg) success = False @@ -1312,7 +1472,8 @@ def scan_codebase( success = False resource.scan_errors.extend(scan_errors) - if TRACE: logger_debug('scan_codebase: scan_timings:', scan_timings) + if TRACE: + logger_debug("scan_codebase: scan_timings:", scan_timings) if with_timing and scan_timings: if scan_timings: resource.scan_timings.update(scan_timings) @@ -1326,8 +1487,8 @@ def scan_codebase( if not value: # the scan attribute will have a default value continue - if key.startswith('extra_data.'): - key = key.replace('extra_data.', '') + if key.startswith("extra_data."): + key = key.replace("extra_data.", "") resource.extra_data[key] = value else: setattr(resource, key, value) @@ -1339,7 +1500,7 @@ def scan_codebase( except StopIteration: break except KeyboardInterrupt: - echo_func('\nAborted with Ctrl+C!', fg='red') + echo_func("\nAborted with Ctrl+C!", fg="red") success = False terminate_pool(pool) break @@ -1349,7 +1510,7 @@ def scan_codebase( # http://bugs.python.org/issue15101 terminate_pool(pool) - if scans and hasattr(scans, 'render_finish'): + if scans and hasattr(scans, "render_finish"): # hack to avoid using a context manager scans.render_finish() return success @@ -1434,14 +1595,14 @@ def scan_resource( runner = partial(scanner.function, location, path=path, deadline=deadline) error, values_mapping = interruptor(runner, timeout=timeout) if error: - msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + error + msg = "ERROR: for scanner: " + scanner.name + ":\n" + error scan_errors.append(msg) # the return value of a scanner fun MUST be a mapping if values_mapping: results.update(values_mapping) except Exception: - msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc() + msg = "ERROR: for scanner: " + scanner.name + ":\n" + traceback.format_exc() scan_errors.append(msg) finally: if with_timing: @@ -1457,9 +1618,10 @@ def display_summary(codebase, scan_names, processes, errors, echo_func=echo_stde Display a scan summary. """ error_messages, summary_messages = get_displayable_summary( - codebase, scan_names, processes, errors) + codebase, scan_names, processes, errors + ) for msg in error_messages: - echo_func(msg, fg='red') + echo_func(msg, fg="red") for msg in summary_messages: echo_func(msg) @@ -1469,46 +1631,50 @@ def get_displayable_summary(codebase, scan_names, processes, errors): """ Return displayable summary messages """ - initial_files_count = codebase.counters.get('initial:files_count', 0) - initial_dirs_count = codebase.counters.get('initial:dirs_count', 0) + initial_files_count = codebase.counters.get("initial:files_count", 0) + initial_dirs_count = codebase.counters.get("initial:dirs_count", 0) initial_res_count = initial_files_count + initial_dirs_count - initial_size_count = codebase.counters.get('initial:size_count', 0) + initial_size_count = codebase.counters.get("initial:size_count", 0) if initial_size_count: initial_size_count = format_size(initial_size_count) - initial_size_count = 'for %(initial_size_count)s' % locals() + initial_size_count = "for %(initial_size_count)s" % locals() else: - initial_size_count = '' + initial_size_count = "" ###################################################################### - prescan_scan_time = codebase.timings.get('pre-scan-scan', 0.) + prescan_scan_time = codebase.timings.get("pre-scan-scan", 0.0) if prescan_scan_time: - prescan_scan_files_count = codebase.counters.get('pre-scan-scan:files_count', 0) - prescan_scan_file_speed = round(float(prescan_scan_files_count) / prescan_scan_time , 2) + prescan_scan_files_count = codebase.counters.get("pre-scan-scan:files_count", 0) + prescan_scan_file_speed = round( + float(prescan_scan_files_count) / prescan_scan_time, 2 + ) - prescan_scan_size_count = codebase.counters.get('pre-scan-scan:size_count', 0) + prescan_scan_size_count = codebase.counters.get("pre-scan-scan:size_count", 0) if prescan_scan_size_count: - prescan_scan_size_speed = format_size(prescan_scan_size_count / prescan_scan_time) - prescan_scan_size_speed = '%(prescan_scan_size_speed)s/sec.' % locals() + prescan_scan_size_speed = format_size( + prescan_scan_size_count / prescan_scan_time + ) + prescan_scan_size_speed = "%(prescan_scan_size_speed)s/sec." % locals() prescan_scan_size_count = format_size(prescan_scan_size_count) - prescan_scan_size_count = 'for %(prescan_scan_size_count)s' % locals() + prescan_scan_size_count = "for %(prescan_scan_size_count)s" % locals() else: - prescan_scan_size_count = '' - prescan_scan_size_speed = '' + prescan_scan_size_count = "" + prescan_scan_size_speed = "" ###################################################################### - scan_time = codebase.timings.get('scan', 0.) + scan_time = codebase.timings.get("scan", 0.0) - scan_files_count = codebase.counters.get('scan:files_count', 0) + scan_files_count = codebase.counters.get("scan:files_count", 0) if scan_time: - scan_file_speed = round(float(scan_files_count) / scan_time , 2) + scan_file_speed = round(float(scan_files_count) / scan_time, 2) else: scan_file_speed = 0 - scan_size_count = codebase.counters.get('scan:size_count', 0) + scan_size_count = codebase.counters.get("scan:size_count", 0) if scan_size_count: if scan_time: @@ -1516,31 +1682,31 @@ def get_displayable_summary(codebase, scan_names, processes, errors): else: scan_size_speed = 0 - scan_size_speed = '%(scan_size_speed)s/sec.' % locals() + scan_size_speed = "%(scan_size_speed)s/sec." % locals() scan_size_count = format_size(scan_size_count) - scan_size_count = 'for %(scan_size_count)s' % locals() + scan_size_count = "for %(scan_size_count)s" % locals() else: - scan_size_count = '' - scan_size_speed = '' + scan_size_count = "" + scan_size_speed = "" ###################################################################### - final_files_count = codebase.counters.get('final:files_count', 0) - final_dirs_count = codebase.counters.get('final:dirs_count', 0) + final_files_count = codebase.counters.get("final:files_count", 0) + final_dirs_count = codebase.counters.get("final:dirs_count", 0) final_res_count = final_files_count + final_dirs_count - final_size_count = codebase.counters.get('final:size_count', 0) + final_size_count = codebase.counters.get("final:size_count", 0) if final_size_count: final_size_count = format_size(final_size_count) - final_size_count = 'for %(final_size_count)s' % locals() + final_size_count = "for %(final_size_count)s" % locals() else: - final_size_count = '' + final_size_count = "" ###################################################################### error_messages = [] errors_count = len(errors) if errors: - error_messages.append('Some files failed to scan properly:') + error_messages.append("Some files failed to scan properly:") for error in errors: for me in error.splitlines(False): error_messages.append(me) @@ -1548,39 +1714,47 @@ def get_displayable_summary(codebase, scan_names, processes, errors): ###################################################################### summary_messages = [] - summary_messages.append('Summary: %(scan_names)s with %(processes)d process(es)' % locals()) - summary_messages.append('Errors count: %(errors_count)d' % locals()) - summary_messages.append('Scan Speed: %(scan_file_speed).2f files/sec. %(scan_size_speed)s' % locals()) + summary_messages.append( + "Summary: %(scan_names)s with %(processes)d process(es)" % locals() + ) + summary_messages.append("Errors count: %(errors_count)d" % locals()) + summary_messages.append( + "Scan Speed: %(scan_file_speed).2f files/sec. %(scan_size_speed)s" + % locals() + ) if prescan_scan_time: summary_messages.append( - 'Early Scanners Speed: %(prescan_scan_file_speed).2f ' - 'files/sec. %(prescan_scan_size_speed)s' % locals() + "Early Scanners Speed: %(prescan_scan_file_speed).2f " + "files/sec. %(prescan_scan_size_speed)s" % locals() ) summary_messages.append( - 'Initial counts: %(initial_res_count)d resource(s): ' - '%(initial_files_count)d file(s) ' - 'and %(initial_dirs_count)d directorie(s) ' - '%(initial_size_count)s' % locals() + "Initial counts: %(initial_res_count)d resource(s): " + "%(initial_files_count)d file(s) " + "and %(initial_dirs_count)d directorie(s) " + "%(initial_size_count)s" % locals() ) summary_messages.append( - 'Final counts: %(final_res_count)d resource(s): ' - '%(final_files_count)d file(s) ' - 'and %(final_dirs_count)d directorie(s) ' - '%(final_size_count)s' % locals() + "Final counts: %(final_res_count)d resource(s): " + "%(final_files_count)d file(s) " + "and %(final_dirs_count)d directorie(s) " + "%(final_size_count)s" % locals() ) - summary_messages.append('Timings:') + summary_messages.append("Timings:") cle = codebase.get_or_create_current_header().to_dict() - summary_messages.append(' scan_start: {start_timestamp}'.format(**cle)) - summary_messages.append(' scan_end: {end_timestamp}'.format(**cle)) + summary_messages.append(" scan_start: {start_timestamp}".format(**cle)) + summary_messages.append(" scan_end: {end_timestamp}".format(**cle)) - for name, value, in codebase.timings.items(): + for ( + name, + value, + ) in codebase.timings.items(): if value > 0.1: - summary_messages.append(' %(name)s: %(value).2fs' % locals()) + summary_messages.append(" %(name)s: %(value).2fs" % locals()) # TODO: if timing was requested display top per-scan/per-file stats? @@ -1595,17 +1769,18 @@ def collect_errors(codebase, verbose=False): errors = [] errors.extend(codebase.errors or []) - path_with_errors = [(r.path, r.scan_errors) - for r in codebase.walk() if r.scan_errors] + path_with_errors = [ + (r.path, r.scan_errors) for r in codebase.walk() if r.scan_errors + ] for errored_path, path_errors in path_with_errors: - msg = 'Path: ' + errored_path + msg = "Path: " + errored_path if verbose: msg = [msg] for path_error in path_errors: for emsg in path_error.splitlines(False): - msg.append(' ' + emsg) - msg = '\n'.join(msg) + msg.append(" " + emsg) + msg = "\n".join(msg) errors.append(msg) return errors @@ -1628,19 +1803,19 @@ def format_size(size): >>> assert format_size(1024*1024*1024*12.3) == '12.30 GB' """ if not size: - return '0 Byte' + return "0 Byte" if size < 1: - return '%(size).1f Byte' % locals() + return "%(size).1f Byte" % locals() if size == 1: - return '%(size)d Byte' % locals() + return "%(size)d Byte" % locals() size = float(size) - for symbol in ('Bytes', 'KB', 'MB', 'GB', 'TB'): + for symbol in ("Bytes", "KB", "MB", "GB", "TB"): if size < 1024: if int(size) == float(size): - return '%(size)d %(symbol)s' % locals() - return '%(size).2f %(symbol)s' % locals() - size = size / 1024. - return '%(size).2f %(symbol)s' % locals() + return "%(size)d %(symbol)s" % locals() + return "%(size).2f %(symbol)s" % locals() + size = size / 1024.0 + return "%(size).2f %(symbol)s" % locals() def get_pretty_params(ctx, generic_paths=False): @@ -1657,7 +1832,7 @@ def get_pretty_params(ctx, generic_paths=False): """ if TRACE: - logger_debug('get_pretty_params: generic_paths', generic_paths) + logger_debug("get_pretty_params: generic_paths", generic_paths) args = [] options = [] @@ -1669,35 +1844,40 @@ def get_pretty_params(ctx, generic_paths=False): if param.is_eager: continue # This attribute is not yet in Click 6.7 but in head - if getattr(param, 'hidden', False): + if getattr(param, "hidden", False): continue if value == param.default: continue if value is None: continue - if value in (tuple(), [],): + if value in ( + tuple(), + [], + ): # option with multiple values, the value is a tuple continue if isinstance(param.type, click.Path) and generic_paths: - value = '' + value = "" if isinstance(param.type, click.File): if generic_paths: - value = '' + value = "" else: # the value cannot be displayed as-is as this may be an opened file- # like object - vname = getattr(value, 'name', None) + vname = getattr(value, "name", None) if vname: value = vname else: - value = '' + value = "" # coerce to string for non-basic supported types - if not (value in (True, False, None) - or isinstance(value, (str, str, bytes, tuple, list, dict, dict))): + if not ( + value in (True, False, None) + or isinstance(value, (str, str, bytes, tuple, list, dict, dict)) + ): value = repr(value) # opts is a list of CLI options as in "--strip-root": the last opt is @@ -1712,7 +1892,7 @@ def get_pretty_params(ctx, generic_paths=False): return dict(sorted(args) + sorted(options)) -if __name__ == '__main__': +if __name__ == "__main__": # We have this __main__ block so that we can run scancode as a script. # This is needed so we can use the Python debugging features in VSCode. scancode() diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index 93d56ba0d93..e537002af6c 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -121,6 +121,12 @@ def test_run_scan_includes_outdated_in_extra(): assert results['headers'][0]['extra_data']['OUTDATED'] == 'out of date' +def test_no_version_check_run_is_successful(): + test_file = test_env.get_test_loc('single/iproute.c') + result_file = test_env.get_temp_file('json') + run_scan_click(['--no-check-version', test_file, '--json', result_file], expected_rc=0) + + def test_usage_and_help_return_a_correct_script_name_on_all_platforms(): result = run_scan_click(['--help']) assert 'Usage: scancode [OPTIONS]' in result.output