From 95ef639938c8529b77fb44f8f07ab39aba2a883b Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Feb 2021 18:04:49 +0100 Subject: [PATCH 1/8] add a script which syncs the mypy versions --- .github/workflows/sync_linter_versions.py | 54 +++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 .github/workflows/sync_linter_versions.py diff --git a/.github/workflows/sync_linter_versions.py b/.github/workflows/sync_linter_versions.py new file mode 100755 index 00000000000..6df2a00a4a6 --- /dev/null +++ b/.github/workflows/sync_linter_versions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +import argparse +import pathlib + +import yaml +from packaging import version + + +def extract_version(config, name): + repos = config.get("repos") + if repos is None: + raise ValueError("invalid pre-commit configuration") + + for repo in repos: + hooks = repo["hooks"] + hook_names = [hook["id"] for hook in hooks] + if name in hook_names: + return version.parse(repo["rev"]) + + raise KeyError(f"cannot find hook {name!r} in the pre-commit configuration") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dry", action="store_true") + parser.add_argument( + metavar="pre-commit-config", dest="pre_commit_config", type=pathlib.Path + ) + parser.add_argument("requirements", type=pathlib.Path) + args = parser.parse_args() + + with args.pre_commit_config.open() as f: + config = yaml.safe_load(f) + + mypy_version = extract_version(config, "mypy") + + requirements = args.requirements.read_text() + new_requirements = "\n".join( + [ + line if not line.startswith("mypy=") else f"mypy={mypy_version}" + for line in requirements.split("\n") + ] + ) + + if args.dry: + separator = "\n" + "—" * 80 + "\n" + print( + "contents of the new requirements file:", + new_requirements, + sep=separator, + end=separator, + ) + else: + args.requirements.write_text(new_requirements) From 830e1cf8181749783dac0627edf1d45812598b7f Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Feb 2021 18:06:39 +0100 Subject: [PATCH 2/8] call the sync script in the workflow --- .github/workflows/ci-pre-commit-autoupdate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pre-commit-autoupdate.yaml b/.github/workflows/ci-pre-commit-autoupdate.yaml index 784fd05bcb4..41c0186bf38 100644 --- a/.github/workflows/ci-pre-commit-autoupdate.yaml +++ b/.github/workflows/ci-pre-commit-autoupdate.yaml @@ -35,6 +35,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} EXECUTE_COMMANDS: | python -m pre_commit autoupdate + python .github/workflows/sync_linter_versions.py .pre-commit-config.yaml ci/requirements/mypy_only COMMIT_MESSAGE: 'pre-commit: autoupdate hook versions' PR_TITLE: 'pre-commit: autoupdate hook versions' PR_BRANCH_PREFIX: 'pre-commit/' From 2e22cc1d2319dbd5ab4d8977ae1fb8f57c7c62fd Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Feb 2021 18:09:45 +0100 Subject: [PATCH 3/8] make sure the CI uses the bot as committer --- .github/workflows/ci-pre-commit-autoupdate.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pre-commit-autoupdate.yaml b/.github/workflows/ci-pre-commit-autoupdate.yaml index 41c0186bf38..2ff3b4cfaab 100644 --- a/.github/workflows/ci-pre-commit-autoupdate.yaml +++ b/.github/workflows/ci-pre-commit-autoupdate.yaml @@ -37,6 +37,8 @@ jobs: python -m pre_commit autoupdate python .github/workflows/sync_linter_versions.py .pre-commit-config.yaml ci/requirements/mypy_only COMMIT_MESSAGE: 'pre-commit: autoupdate hook versions' + COMMIT_NAME: 'github-actions[bot]' + COMMIT_EMAIL: 'github-actions[bot]@users.noreply.github.com' PR_TITLE: 'pre-commit: autoupdate hook versions' PR_BRANCH_PREFIX: 'pre-commit/' PR_BRANCH_NAME: 'autoupdate-${PR_ID}' From a02421e7a8f340591fc1533caedbc33fed5657b0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 22 Feb 2021 18:13:39 +0100 Subject: [PATCH 4/8] update the installed dependencies --- .github/workflows/ci-pre-commit-autoupdate.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pre-commit-autoupdate.yaml b/.github/workflows/ci-pre-commit-autoupdate.yaml index 2ff3b4cfaab..70904200cf7 100644 --- a/.github/workflows/ci-pre-commit-autoupdate.yaml +++ b/.github/workflows/ci-pre-commit-autoupdate.yaml @@ -25,8 +25,8 @@ jobs: uses: actions/setup-python@v2 - name: upgrade pip run: python -m pip install --upgrade pip - - name: install pre-commit - run: python -m pip install --upgrade pre-commit + - name: install dependencies + run: python -m pip install --upgrade pre-commit pyyaml packaging - name: version info run: python -m pip list - name: autoupdate From 977da0c511f2d180a1c570ddc576cd58591eaeb6 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Feb 2021 15:19:19 +0100 Subject: [PATCH 5/8] parse all hook versions at once --- .github/workflows/sync_linter_versions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sync_linter_versions.py b/.github/workflows/sync_linter_versions.py index 6df2a00a4a6..159f4affcbe 100755 --- a/.github/workflows/sync_linter_versions.py +++ b/.github/workflows/sync_linter_versions.py @@ -1,23 +1,22 @@ #!/usr/bin/env python import argparse +import itertools import pathlib import yaml from packaging import version -def extract_version(config, name): +def extract_versions(config): repos = config.get("repos") if repos is None: raise ValueError("invalid pre-commit configuration") - for repo in repos: - hooks = repo["hooks"] - hook_names = [hook["id"] for hook in hooks] - if name in hook_names: - return version.parse(repo["rev"]) - - raise KeyError(f"cannot find hook {name!r} in the pre-commit configuration") + extracted_versions = ( + ((hook["id"], version.parse(repo["rev"])) for hook in repo["hooks"]) + for repo in repos + ) + return dict(itertools.chain.from_iterable(extracted_versions)) if __name__ == "__main__": @@ -32,7 +31,8 @@ def extract_version(config, name): with args.pre_commit_config.open() as f: config = yaml.safe_load(f) - mypy_version = extract_version(config, "mypy") + versions = extract_versions(config) + mypy_version = versions["mypy"] requirements = args.requirements.read_text() new_requirements = "\n".join( From 4c6511881f1d4e676d1cba50fc460c20a4501add Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Feb 2021 15:20:42 +0100 Subject: [PATCH 6/8] update all requirements, not just mypy --- .github/workflows/sync_linter_versions.py | 32 +++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sync_linter_versions.py b/.github/workflows/sync_linter_versions.py index 159f4affcbe..f2d0560e18e 100755 --- a/.github/workflows/sync_linter_versions.py +++ b/.github/workflows/sync_linter_versions.py @@ -5,6 +5,7 @@ import yaml from packaging import version +from packaging.requirements import Requirement def extract_versions(config): @@ -19,6 +20,19 @@ def extract_versions(config): return dict(itertools.chain.from_iterable(extracted_versions)) +def update_requirement(line, new_versions): + preprocessed = line.replace("=", "==") # convert to pep-508 compatible + requirement = Requirement(preprocessed) + + specifier, *_ = requirement.specifier + old_version = specifier.version + new_version = new_versions.get(requirement.name, old_version) + + new_line = f"{requirement.name}={new_version}" + + return new_line + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--dry", action="store_true") @@ -34,13 +48,15 @@ def extract_versions(config): versions = extract_versions(config) mypy_version = versions["mypy"] - requirements = args.requirements.read_text() - new_requirements = "\n".join( - [ - line if not line.startswith("mypy=") else f"mypy={mypy_version}" - for line in requirements.split("\n") - ] - ) + requirements_text = args.requirements.read_text() + requirements = requirements_text.split("\n") + new_requirements = [ + update_requirement(line, versions) + if line and not line.startswith("# ") + else line + for line in requirements + ] + new_requirements_text = "\n".join(new_requirements) if args.dry: separator = "\n" + "—" * 80 + "\n" @@ -51,4 +67,4 @@ def extract_versions(config): end=separator, ) else: - args.requirements.write_text(new_requirements) + args.requirements.write_text(new_requirements_text) From 2d33f3af6109a4a9332d8b6738580c3a0aed8a7d Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Feb 2021 15:21:01 +0100 Subject: [PATCH 7/8] include the original requirements text in the debug output --- .github/workflows/sync_linter_versions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync_linter_versions.py b/.github/workflows/sync_linter_versions.py index f2d0560e18e..f551713ef08 100755 --- a/.github/workflows/sync_linter_versions.py +++ b/.github/workflows/sync_linter_versions.py @@ -61,8 +61,10 @@ def update_requirement(line, new_versions): if args.dry: separator = "\n" + "—" * 80 + "\n" print( + "contents of the old requirements file:", + requirements_text, "contents of the new requirements file:", - new_requirements, + new_requirements_text, sep=separator, end=separator, ) From 1b4cf0823b076db524c1a846cd25737acf5e6877 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 23 Feb 2021 15:39:52 +0100 Subject: [PATCH 8/8] use a re.sub instead of str.replace --- .github/workflows/sync_linter_versions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync_linter_versions.py b/.github/workflows/sync_linter_versions.py index f551713ef08..cb0b1355c71 100755 --- a/.github/workflows/sync_linter_versions.py +++ b/.github/workflows/sync_linter_versions.py @@ -2,11 +2,14 @@ import argparse import itertools import pathlib +import re import yaml from packaging import version from packaging.requirements import Requirement +operator_re = re.compile("=+") + def extract_versions(config): repos = config.get("repos") @@ -21,7 +24,8 @@ def extract_versions(config): def update_requirement(line, new_versions): - preprocessed = line.replace("=", "==") # convert to pep-508 compatible + # convert to pep-508 compatible + preprocessed = operator_re.sub("==", line) requirement = Requirement(preprocessed) specifier, *_ = requirement.specifier