diff --git a/tests/test_cli.py b/tests/test_cli.py index e0ae0fee..120bdbf2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,7 +23,7 @@ import unittest from io import StringIO -from tests.common import build_temp_workspace, RunContext, temp_workspace +from tests.common import RunContext, build_temp_workspace, temp_workspace from yamllint import cli, config @@ -733,6 +733,63 @@ def test_run_list_files(self): [os.path.join(self.wd, 'a.yaml')] ) + def test_multiple_processes(self): + items = {'a.yaml': os.path.join(self.wd, 'a.yaml'), + 'en.yaml': os.path.join(self.wd, 'en.yaml'), + 'c.yaml': os.path.join(self.wd, 'c.yaml'), + } + + with RunContext(self) as ctx: + cli.run(["-w", "2", "-f", "parsable"] + list(items.values())) + self.assertEqual(ctx.returncode, 1) + + path = items['a.yaml'] + self.assertIn( + f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n', + ctx.stdout + ) + self.assertIn( + f'{path}:3:4: [error] no new line character at the end of file ' + '(new-line-at-end-of-file)\n', + ctx.stdout + ) + + path = items['en.yaml'] + self.assertIn( + f'{path}:3:8: [error] no new line character at the end of file ' + '(new-line-at-end-of-file)\n', + ctx.stdout + ) + + path = items['c.yaml'] + self.assertIn( + f'{path}:3:8: [error] no new line character at the end of file ' + '(new-line-at-end-of-file)\n', + ctx.stdout + ) + + with RunContext(self) as ctx: + cli.run(["-w", "0", "-f", "parsable"] + list(items.values())) + self.assertEqual(ctx.returncode, 1) + + path = items['a.yaml'] + self.assertIn( + f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n', + ctx.stdout + ) + + def test_multiple_processes_non_existing_file(self): + items = {'a.yaml': os.path.join(self.wd, 'a.yaml'), + 'en.yaml': os.path.join(self.wd, 'en.yaml'), + 'c.yaml': os.path.join(self.wd, 'c.yaml'), + 'fails': os.path.join(self.wd, 'i-do-not-exist.yaml') + } + + with RunContext(self) as ctx: + cli.run(["-w", "2", "-f", "parsable"] + list(items.values())) + self.assertEqual(ctx.returncode, -1) + self.assertRegex(ctx.stderr, r'No such file or directory') + class CommandLineConfigTestCase(unittest.TestCase): def test_config_file(self): diff --git a/yamllint/cli.py b/yamllint/cli.py index 9a39bd8c..4c26142b 100644 --- a/yamllint/cli.py +++ b/yamllint/cli.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import argparse +import concurrent.futures import locale import os import platform @@ -143,6 +144,12 @@ def find_project_config_filepath(path='.'): return find_project_config_filepath(path=os.path.join(path, '..')) +def _do_lint(filename, conf): + filepath = filename[2:] if filename.startswith("./") else filename + with open(filename, newline="") as f: + return list(linter.run(f, conf, filepath)) + + def run(argv=None): parser = argparse.ArgumentParser(prog=APP_NAME, description=APP_DESCRIPTION) @@ -172,6 +179,12 @@ def run(argv=None): parser.add_argument('--no-warnings', action='store_true', help='output only error level problems') + parser.add_argument('-w', '--workers', + action='store', + type=int, + default=1, + help='maximum number of parallel processes to run ' + '(0 for auto, 1 for single-process)') parser.add_argument('-v', '--version', action='version', version=f'{APP_NAME} {APP_VERSION}') @@ -216,17 +229,45 @@ def run(argv=None): max_level = 0 - for file in find_files_recursively(args.files, conf): - filepath = file[2:] if file.startswith('./') else file - try: - with open(file, newline='') as f: - problems = linter.run(f, conf, filepath) - except OSError as e: - print(e, file=sys.stderr) - sys.exit(-1) - prob_level = show_problems(problems, file, args_format=args.format, - no_warn=args.no_warnings) - max_level = max(max_level, prob_level) + if args.workers == 0: + args.workers = None + + if args.workers == 1: + print('here') + for file in find_files_recursively(args.files, conf): + filepath = file[2:] if file.startswith('./') else file + try: + with open(file, newline='') as f: + problems = linter.run(f, conf, filepath) + except OSError as e: + print(e, file=sys.stderr) + sys.exit(-1) + prob_level = show_problems(problems, file, args_format=args.format, + no_warn=args.no_warnings) + max_level = max(max_level, prob_level) + else: + with concurrent.futures.ProcessPoolExecutor( + max_workers=args.workers + ) as executor: + futures = { + executor.submit(_do_lint, file, conf): file + for file in find_files_recursively(args.files, conf) + } + + for future in concurrent.futures.as_completed(futures): + try: + problems = future.result() + except OSError as e: + print(e, file=sys.stderr) + sys.exit(-1) + + prob_level = show_problems( + problems, + futures[future], + args_format=args.format, + no_warn=args.no_warnings, + ) + max_level = max(max_level, prob_level) # read yaml from stdin if args.stdin: