Skip to content

Commit

Permalink
LineEndingsCheck: improve performance
Browse files Browse the repository at this point in the history
LineEndingsCheck generally takes anywhere from 4 to 6 seconds to run,
depending on the size of the package. This change reduces the time to
run by ~53% as tested on packages in MU_BASECORE due to the following
changes:

1. Reorganized filters to reduce the amount of list comprehensions
   performed to filter out files that do not need to be checked

2. Organized the order of checks in regards to filters most likely to
   catch a file, and the expensiveness of the filter itself.
  • Loading branch information
Javagedes committed Aug 8, 2023
1 parent c69d5a7 commit 0982bd9
Showing 1 changed file with 16 additions and 37 deletions.
53 changes: 16 additions & 37 deletions .pytool/Plugin/LineEndingCheck/LineEndingCheck.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,47 +260,27 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path,
tc.LogStdError(f"Package folder not found {self._abs_pkg_path}")
return 0

all_files = [Path(n) for n in glob.glob(
os.path.join(self._abs_pkg_path, '**/*.*'),
recursive=True)]
ignored_files = list(filter(
self._get_files_ignored_in_config(
package_config, self._abs_pkg_path), all_files))
ignored_files = [Path(f) for f in ignored_files]

all_files = list(set(all_files) - set(ignored_files))
if not all_files:
tc.SetSuccess()
return 0

all_files_before_git_removal = set(all_files)
git_ignored_paths = set(self._get_git_ignored_paths() + self._get_git_submodule_paths())
all_files = list(all_files_before_git_removal - git_ignored_paths)
git_ignored_paths = git_ignored_paths - (all_files_before_git_removal - set(all_files))
if not all_files:
tc.SetSuccess()
return 0

git_ignored_paths = {p for p in git_ignored_paths if p.is_dir()}

ignored_files = []
for file in all_files:
for ignored_path in git_ignored_paths:
if Path(file).is_relative_to(ignored_path):
ignored_files.append(file)
break

all_files = list(set(all_files) - set(ignored_files))
if not all_files:
tc.SetSuccess()
return 0
# MU_CHANGE begin: Perf Improvements
ignore_files = set(self._get_git_ignored_paths())
ignore_dirs = set(self._get_git_submodule_paths())
ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path)

file_count = 0
line_ending_count = dict.fromkeys(LINE_ENDINGS, 0)

for file in all_files:
for file in Path(self._abs_pkg_path).rglob('*'):
if file.is_dir():
continue

if any(file.is_relative_to(ignore_dir) for ignore_dir in ignore_dirs):
continue

if ignore_filter(file):
continue

if file in ignore_files:
continue

# MU_CHANGE end: Perf Improvements
with open(file.resolve(), 'rb') as fb:
if not fb.readable() or _is_binary_string(fb.read(1024)):
continue
Expand Down Expand Up @@ -341,5 +321,4 @@ def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path,
"CHECK_FAILED")
else:
tc.SetSuccess()

return sum(line_ending_count.values())

0 comments on commit 0982bd9

Please sign in to comment.