diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d2a320af2b..fa99cbf88ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-linter.yml file, or with `oxsecurity/megalinter:beta` docker image - Core + - Allow to override the number of parallel cores used, with variable **PARALLEL_PROCESS_NUMBER** - Media - New linters - Fixes + - Fix conflict between prettier and yamllint about spaces - Doc diff --git a/README.md b/README.md index 0b03b2030c1..4d0d780f43b 100644 --- a/README.md +++ b/README.md @@ -981,6 +981,7 @@ description: List of common variables that you can use to customize MegaLinter b | **MEGALINTER_CONFIG** | `.mega-linter.yml` | Name of MegaLinter configuration file. Can be defined remotely, in that case set this environment variable with the remote URL of `.mega-linter.yml` config file | | **MEGALINTER_FILES_TO_LINT** | \[\] | Comma-separated list of files to analyze. Using this variable will bypass other file listing methods | | **PARALLEL** | `true` | Process linters in parallel to improve overall MegaLinter performance. If true, linters of same language or formats are grouped in the same parallel process to avoid lock issues if fixing the same files | +| **PARALLEL_PROCESS_NUMBER** | | All available cores are used by default. If there are too many, you need to decrease the number of used cores in order to enhance performances (example: `4`) | | [**PLUGINS**](#plugins) | \[\] | List of plugin urls to install and run during MegaLinter run | | [**POST_COMMANDS**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-postcommands.md) | \[\] | Custom bash commands to run after linters | | [**PRE_COMMANDS**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-precommands.md) | \[\] | Custom bash commands to run before linters | diff --git a/megalinter/MegaLinter.py b/megalinter/MegaLinter.py index 2505f2a96e0..1e870ed2633 100644 --- a/megalinter/MegaLinter.py +++ b/megalinter/MegaLinter.py @@ -318,8 +318,12 @@ def process_linters_parallel(self, active_linters, linters_do_fixes): linter_groups += [[linter]] linter_groups = linter_factory.sort_linters_groups_by_speed(linter_groups) # Execute linters in asynchronous pool to improve overall performances - process_number = mp.cpu_count() - logging.info(f"Processing linters on [{str(process_number)}] parallel cores…") + if config.exists("PARALLEL_PROCESS_NUMBER"): + process_number = int(config.get("PARALLEL_PROCESS_NUMBER")) + logging.info(f"Processing linters on [{str(process_number)}] parallel cores… (according to variable PARALLEL_PROCESS_NUMBER") + else: + process_number = mp.cpu_count() + logging.info(f"Processing linters on [{str(process_number)}] parallel cores… (can be decreased with variable PARALLEL_PROCESS_NUMBER in case of performance issues)") install_mp_handler() pool = mp.Pool( process_number, diff --git a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json index 9d0cf16be3d..0eb1604af5b 100644 --- a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json @@ -10242,6 +10242,15 @@ "title": "Parallel processing", "type": "boolean" }, + "PARALLEL_PROCESS_NUMBER": { + "$id": "#/properties/PARALLEL_PROCESS_NUMBER", + "description": "All available cores are used by default. If there are too many, you need to decrease the number of used cores in order to enhance performances", + "examples": [ + 4 + ], + "title": "Parallel process number", + "type": "number" + }, "PERL_FILTER_REGEX_EXCLUDE": { "$id": "#/properties/PERL_FILTER_REGEX_EXCLUDE", "title": "Excluding regex filter for PERL descriptor",