From 0ce508bdbe1a81d497e4c9133fb7b52f553a9883 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 8 Aug 2023 09:11:45 -0700 Subject: [PATCH 1/5] LineEndingsCheck: improve performance LineEndingsCheck generally takes anywhere from 4 to 6 seconds to run, depending on the size of the package. This change reduces the time to run by ~53% as tested on packages in MU_BASECORE due to the following changes: 1. Reorganized filters to reduce the amount of list comprehensions performed to filter out files that do not need to be checked 2. Organized the order of checks in regards to filters most likely to catch a file, and the expensiveness of the filter itself. --- .../Plugin/LineEndingCheck/LineEndingCheck.py | 52 ++++++------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py index e34ed62414..12faafa959 100644 --- a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py +++ b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py @@ -260,47 +260,27 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, tc.LogStdError(f"Package folder not found {self._abs_pkg_path}") return 0 - all_files = [Path(n) for n in glob.glob( - os.path.join(self._abs_pkg_path, '**/*.*'), - recursive=True)] - ignored_files = list(filter( - self._get_files_ignored_in_config( - package_config, self._abs_pkg_path), all_files)) - ignored_files = [Path(f) for f in ignored_files] - - all_files = list(set(all_files) - set(ignored_files)) - if not all_files: - tc.SetSuccess() - return 0 - - all_files_before_git_removal = set(all_files) - git_ignored_paths = set(self._get_git_ignored_paths() + self._get_git_submodule_paths()) - all_files = list(all_files_before_git_removal - git_ignored_paths) - git_ignored_paths = git_ignored_paths - (all_files_before_git_removal - set(all_files)) - if not all_files: - tc.SetSuccess() - return 0 - - git_ignored_paths = {p for p in git_ignored_paths if p.is_dir()} - - ignored_files = [] - for file in all_files: - for ignored_path in git_ignored_paths: - if Path(file).is_relative_to(ignored_path): - ignored_files.append(file) - break - - all_files = list(set(all_files) - set(ignored_files)) - if not all_files: - tc.SetSuccess() - return 0 + # MU_CHANGE begin: Perf Improvements + ignore_files = set(self._get_git_ignored_paths()) + ignore_dirs = set(self._get_git_submodule_paths()) + ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path) file_count = 0 line_ending_count = dict.fromkeys(LINE_ENDINGS, 0) - - for file in all_files: + for file in Path(self._abs_pkg_path).rglob('*'): if file.is_dir(): continue + + if any(file.is_relative_to(ignore_dir) for ignore_dir in ignore_dirs): + continue + + if ignore_filter(file): + continue + + if file in ignore_files: + continue + + # MU_CHANGE end: Perf Improvements with open(file.resolve(), 'rb') as fb: if not fb.readable() or _is_binary_string(fb.read(1024)): continue From 512d8671374ffed7bc91ffd4259675e9f3d4dfe4 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 9 Aug 2023 10:46:10 -0700 Subject: [PATCH 2/5] Update to support ignoring files with no extension --- .pytool/Plugin/LineEndingCheck/LineEndingCheck.py | 6 +++--- .pytool/Plugin/LineEndingCheck/Readme.md | 6 ++++++ BaseTools/BaseTools.ci.yaml | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py index 12faafa959..8eb5c7e42e 100644 --- a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py +++ b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py @@ -207,8 +207,10 @@ def _get_files_ignored_in_config(self, Callable[[None], None]: A test case function. """ ignored_files = [] + if pkg_config.get("IgnoreFilesWithNoExtension", False): + ignored_files.extend(['*', '!*.*', '!*/']) if "IgnoreFiles" in pkg_config: - ignored_files = pkg_config["IgnoreFiles"] + ignored_files.extend(pkg_config["IgnoreFiles"]) # Pass "Package configuration file" as the source file path since # the actual configuration file name is unknown to this plugin and @@ -260,7 +262,6 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, tc.LogStdError(f"Package folder not found {self._abs_pkg_path}") return 0 - # MU_CHANGE begin: Perf Improvements ignore_files = set(self._get_git_ignored_paths()) ignore_dirs = set(self._get_git_submodule_paths()) ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path) @@ -280,7 +281,6 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, if file in ignore_files: continue - # MU_CHANGE end: Perf Improvements with open(file.resolve(), 'rb') as fb: if not fb.readable() or _is_binary_string(fb.read(1024)): continue diff --git a/.pytool/Plugin/LineEndingCheck/Readme.md b/.pytool/Plugin/LineEndingCheck/Readme.md index dca1b47940..384cab71d5 100644 --- a/.pytool/Plugin/LineEndingCheck/Readme.md +++ b/.pytool/Plugin/LineEndingCheck/Readme.md @@ -19,9 +19,15 @@ The plugin can be configured to ignore certain files. ``` yaml "LineEndingCheck": { "IgnoreFiles": [] + "IgnoreFilesWithNoExtension": False } ``` ### IgnoreFiles An **optional** list of git ignore patterns relative to the package root used to exclude files from being checked. + +### IgnoreFilesWithNoExtension + +An **optional** value that, if True, will insert the gitignore rules necessary to have this check ignore files +that do not contain a file extension. Neccessary for binary files and/or POSIX like executables. diff --git a/BaseTools/BaseTools.ci.yaml b/BaseTools/BaseTools.ci.yaml index bc071efa06..5b4198915d 100644 --- a/BaseTools/BaseTools.ci.yaml +++ b/BaseTools/BaseTools.ci.yaml @@ -44,7 +44,7 @@ "AuditOnly": True, }, "LineEndingCheck": { - "IgnoreFiles": ['*.*'] + "IgnoreFiles": ["*"], } # MU_CHANGE [END] } From b85c473421307ec3e8a3046d01472e8a66e62a31 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 9 Aug 2023 10:47:15 -0700 Subject: [PATCH 3/5] Update spelling --- .pytool/Plugin/LineEndingCheck/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pytool/Plugin/LineEndingCheck/Readme.md b/.pytool/Plugin/LineEndingCheck/Readme.md index 384cab71d5..5699f27aa0 100644 --- a/.pytool/Plugin/LineEndingCheck/Readme.md +++ b/.pytool/Plugin/LineEndingCheck/Readme.md @@ -30,4 +30,4 @@ An **optional** list of git ignore patterns relative to the package root used to ### IgnoreFilesWithNoExtension An **optional** value that, if True, will insert the gitignore rules necessary to have this check ignore files -that do not contain a file extension. Neccessary for binary files and/or POSIX like executables. +that do not contain a file extension. Necessary for binary files and/or POSIX like executables. From e593b42832121bbe745603d91e056ece250df95d Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 8 Aug 2023 09:11:45 -0700 Subject: [PATCH 4/5] LineEndingsCheck: improve performance LineEndingsCheck generally takes anywhere from 4 to 6 seconds to run, depending on the size of the package. This change reduces the time to run by ~53% as tested on packages in MU_BASECORE due to the following changes: 1. Reorganized filters to reduce the amount of list comprehensions performed to filter out files that do not need to be checked 2. Organized the order of checks in regards to filters most likely to catch a file, and the expensiveness of the filter itself. --- .pytool/Plugin/LineEndingCheck/LineEndingCheck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py index 8eb5c7e42e..ee01d8f5dc 100644 --- a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py +++ b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py @@ -262,6 +262,7 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, tc.LogStdError(f"Package folder not found {self._abs_pkg_path}") return 0 + # MU_CHANGE begin: Perf Improvements ignore_files = set(self._get_git_ignored_paths()) ignore_dirs = set(self._get_git_submodule_paths()) ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path) @@ -281,6 +282,7 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, if file in ignore_files: continue + # MU_CHANGE end: Perf Improvements with open(file.resolve(), 'rb') as fb: if not fb.readable() or _is_binary_string(fb.read(1024)): continue From f8f39bc24433777a178cfd5e0a7689aa710fa603 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 9 Aug 2023 10:46:10 -0700 Subject: [PATCH 5/5] Update to support ignoring files with no extension --- .pytool/Plugin/LineEndingCheck/LineEndingCheck.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py index ee01d8f5dc..8eb5c7e42e 100644 --- a/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py +++ b/.pytool/Plugin/LineEndingCheck/LineEndingCheck.py @@ -262,7 +262,6 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, tc.LogStdError(f"Package folder not found {self._abs_pkg_path}") return 0 - # MU_CHANGE begin: Perf Improvements ignore_files = set(self._get_git_ignored_paths()) ignore_dirs = set(self._get_git_submodule_paths()) ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path) @@ -282,7 +281,6 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path, if file in ignore_files: continue - # MU_CHANGE end: Perf Improvements with open(file.resolve(), 'rb') as fb: if not fb.readable() or _is_binary_string(fb.read(1024)): continue