From 07cc79f72321eaad38f2a8c4b359c0218327d865 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 27 May 2023 19:30:29 +0100 Subject: [PATCH] Migrated `tasks/filemap.py` -> `tools/precommit/filemap.py` Refs #64374 Signed-off-by: Pedro Algarvio --- changelog/64374.fixed.md | 1 + tasks/filemap.py | 95 -------------------------------------- tools/__init__.py | 3 ++ tools/precommit/filemap.py | 91 ++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 95 deletions(-) delete mode 100644 tasks/filemap.py create mode 100644 tools/precommit/filemap.py diff --git a/changelog/64374.fixed.md b/changelog/64374.fixed.md index 8b94be869d76..e56ef8030360 100644 --- a/changelog/64374.fixed.md +++ b/changelog/64374.fixed.md @@ -3,3 +3,4 @@ Migrated some [`invoke`](https://www.pyinvoke.org/) tasks to [`python-tools-scri * `tasks/docs.py` -> `tools/precommit/docs.py` * `tasks/docstrings.py` -> `tools/precommit/docstrings.py` * `tasks/loader.py` -> `tools/precommit/loader.py` +* `tasks/filemap.py` -> `tools/precommit/filemap.py` diff --git a/tasks/filemap.py b/tasks/filemap.py deleted file mode 100644 index a1eb62c6b82f..000000000000 --- a/tasks/filemap.py +++ /dev/null @@ -1,95 +0,0 @@ -""" - tasks.filemap - ~~~~~~~~~~~~~ - - tests/filename_map.yml validity checks -""" -import pathlib -import re - -import yaml -from invoke import task # pylint: disable=3rd-party-module-not-gated - -from tasks import utils - -CODE_DIR = pathlib.Path(__file__).resolve().parent.parent -FILENAME_MAP_PATH = CODE_DIR / "tests" / "filename_map.yml" - - -def _match_to_test_file(match): - tests_path = CODE_DIR / "tests" - parts = match.split(".") - parts[-1] += ".py" - return tests_path.joinpath(*parts).relative_to(CODE_DIR) - - -def _check_matches(rule, matches): - errors = 0 - for match in matches: - filematch = _match_to_test_file(match) - if not filematch.exists(): - utils.error( - "The match '{}' for rule '{}' points to a non existing test module" - " path: {}", - match, - rule, - filematch, - ) - errors += 1 - return errors - - -@task -def check(ctx): - exitcode = 0 - excludes = ("tasks/", "templates/", ".nox/") - full_filelist = [path.relative_to(CODE_DIR) for path in CODE_DIR.rglob("*.py")] - filelist = [ - str(path) for path in full_filelist if not str(path).startswith(excludes) - ] - filename_map = yaml.safe_load(FILENAME_MAP_PATH.read_text()) - checked = set() - for rule, matches in filename_map.items(): - if rule == "*": - exitcode += _check_matches(rule, matches) - elif "|" in rule: - # This is regex - for filepath in filelist: - if re.match(rule, filepath): - # Found at least one match, stop looking - break - else: - utils.error( - "Could not find a matching file in the salt repo for the rule '{}'", - rule, - ) - exitcode += 1 - continue - exitcode += _check_matches(rule, matches) - elif "*" in rule or "\\" in rule: - # Glob matching - process_matches = True - for filerule in CODE_DIR.glob(rule): - if not filerule.exists(): - utils.error( - "The rule '{}' points to a non existing path: {}", - rule, - filerule, - ) - exitcode += 1 - process_matches = False - if process_matches: - exitcode += _check_matches(rule, matches) - else: - # Direct file paths as rules - filerule = pathlib.Path(rule) - if not filerule.exists(): - utils.error( - "The rule '{}' points to a non existing path: {}", rule, filerule - ) - exitcode += 1 - continue - exitcode += _check_matches(rule, matches) - if exitcode: - utils.error("Found {} errors", exitcode) - utils.exit_invoke(exitcode) diff --git a/tools/__init__.py b/tools/__init__.py index 88f533eb4a86..9c317b687e76 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -52,6 +52,9 @@ ptscripts.register_tools_module( "tools.precommit.docstrings", venv_config=SALT_PKG_DEPS_VENV_CONFIG ) +ptscripts.register_tools_module( + "tools.precommit.filemap", venv_config=SALT_PKG_DEPS_VENV_CONFIG +) ptscripts.register_tools_module( "tools.precommit.loader", venv_config=SALT_PKG_DEPS_VENV_CONFIG ) diff --git a/tools/precommit/filemap.py b/tools/precommit/filemap.py new file mode 100644 index 000000000000..96a662fa7e7e --- /dev/null +++ b/tools/precommit/filemap.py @@ -0,0 +1,91 @@ +""" +`tests/filename_map.yml` validity checks +""" +import pathlib +import re + +import yaml +from ptscripts import Context, command_group + +import tools.utils + +FILENAME_MAP_PATH = tools.utils.REPO_ROOT / "tests" / "filename_map.yml" + +cgroup = command_group(name="filemap", help=__doc__, parent="pre-commit") + + +def _match_to_test_file(match: str) -> pathlib.Path: + tests_path = tools.utils.REPO_ROOT / "tests" + parts = match.split(".") + parts[-1] += ".py" + return tests_path.joinpath(*parts).relative_to(tools.utils.REPO_ROOT) + + +def _check_matches(ctx: Context, rule: str, matches: list[str]) -> int: + errors = 0 + for match in matches: + filematch = _match_to_test_file(match) + if not filematch.exists(): + ctx.error( + f"The match '{match}' for rule '{rule}' points to a non " + f"existing test module path: {filematch}" + ) + errors += 1 + return errors + + +@cgroup.command( + name="check", +) +def check(ctx: Context) -> None: + exitcode = 0 + excludes = ("tools/", "templates/", ".nox/") + full_filelist = [ + path.relative_to(tools.utils.REPO_ROOT) + for path in tools.utils.REPO_ROOT.rglob("*.py") + ] + filelist = [ + str(path) for path in full_filelist if not str(path).startswith(excludes) + ] + filename_map = yaml.safe_load(FILENAME_MAP_PATH.read_text()) + for rule, matches in filename_map.items(): + if rule == "*": + exitcode += _check_matches(ctx, rule, matches) + elif "|" in rule: + # This is regex + for filepath in filelist: + if re.match(rule, filepath): + # Found at least one match, stop looking + break + else: + ctx.error( + f"Could not find a matching file in the salt repo for the rule '{rule}'" + ) + exitcode += 1 + continue + exitcode += _check_matches(ctx, rule, matches) + elif "*" in rule or "\\" in rule: + # Glob matching + process_matches = True + for filerule in tools.utils.REPO_ROOT.glob(rule): + if not filerule.exists(): + ctx.error( + f"The rule '{rule}' points to a non existing path: {filerule}" + ) + exitcode += 1 + process_matches = False + if process_matches: + exitcode += _check_matches(ctx, rule, matches) + else: + # Direct file paths as rules + filerule = pathlib.Path(rule) + if not filerule.exists(): + ctx.error( + f"The rule '{rule}' points to a non existing path: {filerule}" + ) + exitcode += 1 + continue + exitcode += _check_matches(ctx, rule, matches) + if exitcode: + ctx.error(f"Found {exitcode} errors") + ctx.exit(exitcode)