diff --git a/.automation/generated/megalinter-users.json b/.automation/generated/megalinter-users.json index 3388c8459ea..92eb354dacb 100644 --- a/.automation/generated/megalinter-users.json +++ b/.automation/generated/megalinter-users.json @@ -63,8 +63,8 @@ "network_count": 858, "node_id": "MDEwOlJlcG9zaXRvcnkxNjM4ODMyNzk=", "notifications_url": "https://api.github.com/repos/nektos/act/notifications{?since,all,participating}", - "open_issues": 89, - "open_issues_count": 89, + "open_issues": 90, + "open_issues_count": 90, "organization": { "avatar_url": "https://avatars.githubusercontent.com/u/3813618?v=4", "events_url": "https://api.github.com/users/nektos/events{/privacy}", diff --git a/.automation/test/pre-post-test/README.md b/.automation/test/pre-post-test/README.md new file mode 100644 index 00000000000..f15dcfcef4c --- /dev/null +++ b/.automation/test/pre-post-test/README.md @@ -0,0 +1,3 @@ +# Hello + +World diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6e25448be..d152005cfed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - Display a different message in log when a Merge Request comment is created or updated. - Core + - Use relative file paths to call linters ([#1875](https://github.com/oxsecurity/megalinter/issues/1875)) - Add support for idea plugins autoinstall - Upgrade base Docker image to python:3.11.3-alpine3.17 - Fix issue preventing plugins to work with flavors diff --git a/megalinter/Linter.py b/megalinter/Linter.py index 69f7d140d1b..6f3f199eed7 100644 --- a/megalinter/Linter.py +++ b/megalinter/Linter.py @@ -483,6 +483,11 @@ def load_config_vars(self, params): self.config_file = ( self.default_rules_location + os.path.sep + self.config_file_name ) + # Make config file path absolute if not located in workspace + if self.config_file is not None and not os.path.isfile( + self.workspace + os.path.sep + self.config_file + ): + self.config_file = os.path.abspath(self.config_file) # Set config file label if not set by remote rule if self.config_file is not None and self.config_file_label is None: self.config_file_label = self.config_file.replace( @@ -787,7 +792,7 @@ def collect_files(self, all_files): file_contains_regex_extensions=self.file_contains_regex_extensions, files_sub_directory=self.files_sub_directory, lint_all_other_linters_files=self.lint_all_other_linters_files, - prefix=self.workspace, + workspace=self.workspace, ) self.files_number = len(self.files) logging.debug( @@ -814,12 +819,7 @@ def process_linter(self, file=None): # Execute a linting command . Can be overridden for special cases, like use of PowerShell script # noinspection PyMethodMayBeStatic def execute_lint_command(self, command): - cwd = ( - os.getcwd() - if self.cli_lint_mode in ["file", "list_of_files"] - else self.workspace - ) - cwd = os.path.abspath(cwd) + cwd = os.path.abspath(self.workspace) logging.debug(f"[{self.linter_name}] CWD: {cwd}") subprocess_env = {**os.environ, "FORCE_COLOR": "0"} if type(command) == str: diff --git a/megalinter/MegaLinter.py b/megalinter/MegaLinter.py index c9e5418c677..7540598fa41 100644 --- a/megalinter/MegaLinter.py +++ b/megalinter/MegaLinter.py @@ -614,6 +614,7 @@ def collect_files(self): file_extensions=self.file_extensions, ignored_files=ignored_files, ignore_generated_files=self.ignore_generated_files, + workspace=self.workspace, ) logging.info( @@ -663,7 +664,7 @@ def list_files_git_diff(self): all_files = list() for diff_line in diff.splitlines(): if os.path.isfile(self.workspace + os.path.sep + diff_line): - all_files += [self.workspace + os.path.sep + diff_line] + all_files += [diff_line] return all_files def list_files_all(self): @@ -672,7 +673,7 @@ def list_files_all(self): "Listing all files in directory [" + self.workspace + "], then filter with:" ) all_files = [ - os.path.join(self.workspace, file) + file for file in sorted(os.listdir(self.workspace)) if os.path.isfile(os.path.join(self.workspace, file)) ] @@ -681,8 +682,11 @@ def list_files_all(self): excluded_directories = utils.get_excluded_directories() for dirpath, dirnames, filenames in os.walk(self.workspace, topdown=True): dirnames[:] = [d for d in dirnames if d not in excluded_directories] - all_files += [os.path.join(dirpath, file) for file in sorted(filenames)] - return all_files + all_files += [ + os.path.relpath(os.path.join(dirpath, file), self.workspace) + for file in sorted(filenames) + ] + return list(dict.fromkeys(all_files)) def list_git_ignored_files(self): dirpath = os.path.realpath(self.github_workspace) diff --git a/megalinter/setup.py b/megalinter/setup.py index 120a8068c34..b1ecb6618d1 100644 --- a/megalinter/setup.py +++ b/megalinter/setup.py @@ -21,7 +21,7 @@ "pychalk", "pygithub", "python-gitlab", - "azure-devops", + "azure-devops==6.0.0b4", "commentjson", "pytablewriter", "pyyaml", diff --git a/megalinter/tests/test_megalinter/filters_test.py b/megalinter/tests/test_megalinter/filters_test.py index 6494c7488f6..ed7f05e7112 100644 --- a/megalinter/tests/test_megalinter/filters_test.py +++ b/megalinter/tests/test_megalinter/filters_test.py @@ -7,7 +7,6 @@ import unittest from megalinter import utils -from megalinter.constants import DEFAULT_DOCKER_WORKSPACE_DIR class utilsTest(unittest.TestCase): @@ -50,22 +49,20 @@ def test_file_is_generated_false_2(self): ) def test_filter_files_with_ignored_files(self): - basedir = DEFAULT_DOCKER_WORKSPACE_DIR + "/" all_files = [ - f"{basedir}src/foo.ext", - f"{basedir}README.md", - f"{basedir}target/foo.ext", + "src/foo.ext", + "README.md", + "target/foo.ext", ] for ignored_files, expected in [ ([], all_files), (["hello"], all_files), - (["target/foo.ext"], all_files), + (["target/foo.ext2"], all_files), ( - [f"{basedir}target/foo.ext"], - [f"{basedir}src/foo.ext", f"{basedir}README.md"], + ["target/foo.ext"], + ["src/foo.ext", "README.md"], ), - (["target/**"], all_files), - ([f"{basedir}target/**"], [f"{basedir}src/foo.ext", f"{basedir}README.md"]), + (["target/**"], ["src/foo.ext", "README.md"]), (["foo.ext"], all_files), ]: filtered_files = utils.filter_files( @@ -82,19 +79,18 @@ def test_filter_files_with_ignored_files(self): ) def test_filter_files_with_file_extensions(self): - basedir = DEFAULT_DOCKER_WORKSPACE_DIR + "/" all_files = [ - f"{basedir}src/foo.ext", - f"{basedir}README.md", - f"{basedir}LICENSE", - f"{basedir}target/foo.ext", + "src/foo.ext", + "README.md", + "LICENSE", + "target/foo.ext", ] for file_extensions, expected in [ ([], []), - ([".md"], [f"{basedir}README.md"]), - ([""], [f"{basedir}LICENSE"]), - (["", ".md"], [f"{basedir}LICENSE", f"{basedir}README.md"]), + ([".md"], ["README.md"]), + ([""], ["LICENSE"]), + (["", ".md"], ["LICENSE", "README.md"]), ]: filtered_files = utils.filter_files( all_files=all_files, diff --git a/megalinter/utils.py b/megalinter/utils.py index 36e5076a642..f743cb38f10 100644 --- a/megalinter/utils.py +++ b/megalinter/utils.py @@ -75,7 +75,7 @@ def filter_files( file_contains_regex_extensions: Optional[Sequence[str]] = None, files_sub_directory: Optional[str] = None, lint_all_other_linters_files: bool = False, - prefix: Optional[str] = None, + workspace: str = "", ) -> Sequence[str]: file_extensions = frozenset(file_extensions) filter_regex_include_object = ( @@ -100,18 +100,13 @@ def filter_files( # Filter all files to keep only the ones matching with the current linter for file in all_files: - file_with_prefix_and_sub_dir = os.path.normpath(file) + file_with_prefix_and_sub_dir = os.path.normpath(file).replace(os.sep, "/") + file_with_workspace = os.path.join(workspace, file_with_prefix_and_sub_dir) file = file_with_prefix_and_sub_dir - if prefix or files_sub_directory: - prefix_and_sub_dir = os.path.normpath( - os.path.join(prefix or "", files_sub_directory or "") + os.path.sep - ) - - if file.startswith(prefix_and_sub_dir): - file = os.path.relpath(file_with_prefix_and_sub_dir, prefix_and_sub_dir) - else: - # Skip if file is not in defined files_sub_directory + # skip file if sub_directory necessary + if files_sub_directory is not None: + if not file.startswith(files_sub_directory): continue # Skip if file is in ignore list @@ -126,10 +121,14 @@ def filter_files( base_file_name = os.path.basename(file) _, file_extension = os.path.splitext(base_file_name) # Skip according to FILTER_REGEX_INCLUDE - if filter_regex_include_object and not filter_regex_include_object.search(file): + if filter_regex_include_object and not filter_regex_include_object.search( + file_with_workspace + ): continue # Skip according to FILTER_REGEX_EXCLUDE - if filter_regex_exclude_object and filter_regex_exclude_object.search(file): + if filter_regex_exclude_object and filter_regex_exclude_object.search( + file_with_workspace + ): continue # Skip according to file extension (only if lint_all_other_linter_files is false or file_extensions is defined) @@ -160,16 +159,14 @@ def filter_files( in file_contains_regex_extensions ) ) - and not file_contains( - file_with_prefix_and_sub_dir, file_contains_regex_object - ) + and not file_contains(file_with_workspace, file_contains_regex_object) ): continue # Skip according to IGNORE_GENERATED_FILES if ( ignore_generated_files is not None and ignore_generated_files is True - and file_is_generated(file_with_prefix_and_sub_dir) + and file_is_generated(file_with_workspace) ): continue diff --git a/requirements.dev.txt b/requirements.dev.txt index e92670ae7dc..ee0b09cb79a 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -6,7 +6,7 @@ multiprocessing_logging pychalk pygithub python-gitlab -azure-devops +azure-devops==6.0.0b4 commentjson pytablewriter pytest-cov