From ac23db68c6e34cbb895b5c8d657e233cf0a48130 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:39:52 +0300 Subject: [PATCH 1/5] feat: Add refurb linter --- examples/adapters/refurb/.lintrunner.toml | 28 +++ lintrunner_adapters/adapters/refurb_linter.py | 164 ++++++++++++++++++ requirements-test.txt | 1 + 3 files changed, 193 insertions(+) create mode 100644 examples/adapters/refurb/.lintrunner.toml create mode 100644 lintrunner_adapters/adapters/refurb_linter.py diff --git a/examples/adapters/refurb/.lintrunner.toml b/examples/adapters/refurb/.lintrunner.toml new file mode 100644 index 0000000..1e10b97 --- /dev/null +++ b/examples/adapters/refurb/.lintrunner.toml @@ -0,0 +1,28 @@ +[[linter]] +code = 'REFURB' +include_patterns = [ + '**/*.py', +] +exclude_patterns = [] +command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'refurb_linter', + '--config=pyproject.toml', + '--show-disable', + '--severity=FURB101:advice', + '--severity=FURB102:warning', + '--', + '@{{PATHSFILE}}' +] +init_command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'pip_init', + '--dry-run={{DRYRUN}}', + 'refurb==1.10.0', +] diff --git a/lintrunner_adapters/adapters/refurb_linter.py b/lintrunner_adapters/adapters/refurb_linter.py new file mode 100644 index 0000000..d242001 --- /dev/null +++ b/lintrunner_adapters/adapters/refurb_linter.py @@ -0,0 +1,164 @@ +"""Refurbish and modernize Python code""" + +from __future__ import annotations + +import argparse +import logging +import re +import sys +from textwrap import dedent + +from lintrunner_adapters import LintMessage, LintSeverity, add_default_options +from lintrunner_adapters._common.lintrunner_common import run_command + +LINTER_CODE = "REFURB" + +RESULTS_RE = re.compile( + r"""(?mx) + ^ + (?P.*?): + (?P\d+): + (?:(?P-?\d+))? + (?:\s\[(?P.*)\]:)? + \s(?P.*) + $ + """ +) + + +def _test_results_re() -> None: + """Doctests. + + >>> def t(s): return RESULTS_RE.search(s).groupdict() + + >>> t(r"main.py:3:17 [FURB109]: Use `in (x, y, z)` instead of `in [x, y, z]`") + ... # doctest: +NORMALIZE_WHITESPACE + {'file': 'main.py', 'line': '3', 'column': '17', 'code': 'FURB109', + 'message': 'Use `in (x, y, z)` instead of `in [x, y, z]`'} + + >>> t(r"main.py:4:5 [FURB101]: Use `y = Path(x).read_text()` instead of `with open(x, ...) as f: y = f.read()`") + ... # doctest: +NORMALIZE_WHITESPACE + {'file': 'main.py', 'line': '4', 'column': '5', 'code': 'FURB101', + 'message': 'Use `y = Path(x).read_text()` instead of `with open(x, ...) as f: y = f.read()`'} + """ + pass + + +def format_lint_message(message: str, code: str, show_disable: bool) -> str: + formatted = f"{message}\n" + if show_disable: + formatted += dedent( + f""" + To disable, use + [tool.refurb] + ignore = [{code}] + or + disable = [{code}] + """ + ) + return formatted + + +def check_files( + filenames: list[str], + severities: dict[str, LintSeverity], + *, + config_file: str, + retries: int, + show_disable: bool, +) -> list[LintMessage]: + try: + proc = run_command( + [sys.executable, "-mrefurb", "--config-file", config_file] + filenames, + retries=retries, + ) + except OSError as err: + return [ + LintMessage( + path=None, + line=None, + char=None, + code=LINTER_CODE, + severity=LintSeverity.ERROR, + name="command-failed", + original=None, + replacement=None, + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), + ) + ] + stdout = str(proc.stdout, "utf-8").strip() + return [ + LintMessage( + path=match["file"], + name=match["code"] or "note", + description=format_lint_message( + match["message"], + match["code"], + show_disable, + ), + line=int(match["line"]), + char=int(match["column"]) + if match["column"] is not None and not match["column"].startswith("-") + else None, + code=LINTER_CODE, + severity=severities.get(match["code"], LintSeverity.ERROR), + original=None, + replacement=None, + ) + for match in RESULTS_RE.finditer(stdout) + ] + + +def main() -> None: + parser = argparse.ArgumentParser( + description=f"refurb wrapper linter. Linter code: {LINTER_CODE}", + fromfile_prefix_chars="@", + ) + parser.add_argument( + "--config-file", + required=True, + help="path to pyproject.toml config file", + ) + parser.add_argument( + "--severity", + action="append", + help="map code to severity (e.g. `FURB109:advice`)", + ) + parser.add_argument( + "--show-disable", + action="store_true", + help="show how to disable a lint message", + ) + add_default_options(parser) + args = parser.parse_args() + + logging.basicConfig( + format="<%(threadName)s:%(levelname)s> %(message)s", + level=logging.NOTSET + if args.verbose + else logging.DEBUG + if len(args.filenames) < 1000 + else logging.INFO, + stream=sys.stderr, + ) + + severities: dict[str, LintSeverity] = {} + if args.severity: + for severity in args.severity: + parts = severity.split(":", 1) + assert len(parts) == 2, f"invalid severity `{severity}`" + severities[parts[0]] = LintSeverity(parts[1]) + + lint_messages = check_files( + args.filenames, + severities, + config_file=args.config_file, + retries=args.retries, + show_disable=args.show_disable, + ) + for lint_message in lint_messages: + lint_message.display() + + +if __name__ == "__main__": + main() diff --git a/requirements-test.txt b/requirements-test.txt index cadf061..3d62355 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,5 @@ # Test dependencies to avoid doctest import errors add-trailing-comma==2.4.0 pyupgrade==3.3.1 +refurb==1.10.0;python_version>="3.10" ufmt==2.0.1 From 336f72b5a453a536ffb053009a1c32f00be75551 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 22 Jan 2023 13:56:27 +0300 Subject: [PATCH 2/5] Address review comments --- lintrunner_adapters/adapters/refurb_linter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lintrunner_adapters/adapters/refurb_linter.py b/lintrunner_adapters/adapters/refurb_linter.py index d242001..2b80e56 100644 --- a/lintrunner_adapters/adapters/refurb_linter.py +++ b/lintrunner_adapters/adapters/refurb_linter.py @@ -6,7 +6,7 @@ import logging import re import sys -from textwrap import dedent +import textwrap from lintrunner_adapters import LintMessage, LintSeverity, add_default_options from lintrunner_adapters._common.lintrunner_common import run_command @@ -47,7 +47,7 @@ def _test_results_re() -> None: def format_lint_message(message: str, code: str, show_disable: bool) -> str: formatted = f"{message}\n" if show_disable: - formatted += dedent( + formatted += textwrap.dedent( f""" To disable, use [tool.refurb] @@ -101,7 +101,7 @@ def check_files( if match["column"] is not None and not match["column"].startswith("-") else None, code=LINTER_CODE, - severity=severities.get(match["code"], LintSeverity.ERROR), + severity=severities.get(match["code"], LintSeverity.ADVICE), original=None, replacement=None, ) @@ -116,7 +116,7 @@ def main() -> None: ) parser.add_argument( "--config-file", - required=True, + default="pyproject.toml", help="path to pyproject.toml config file", ) parser.add_argument( From f7329396f28a621ff042297e237ef7a027886c10 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 22 Jan 2023 14:08:00 +0300 Subject: [PATCH 3/5] Address review comments --- lintrunner_adapters/adapters/refurb_linter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lintrunner_adapters/adapters/refurb_linter.py b/lintrunner_adapters/adapters/refurb_linter.py index 2b80e56..3407ce2 100644 --- a/lintrunner_adapters/adapters/refurb_linter.py +++ b/lintrunner_adapters/adapters/refurb_linter.py @@ -122,7 +122,8 @@ def main() -> None: parser.add_argument( "--severity", action="append", - help="map code to severity (e.g. `FURB109:advice`)", + help="map code to severity (e.g. `FURB109:advice`). " + "This option can be used multiple times.", ) parser.add_argument( "--show-disable", From 059d52cccedf345a2886e4b7a3c353a066229ee8 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 22 Jan 2023 23:17:42 +0300 Subject: [PATCH 4/5] Address review comments --- .lintrunner.toml | 76 +++++++++++++------ lintrunner_adapters/adapters/refurb_linter.py | 6 +- pyproject.toml | 3 + 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index 37a3818..f58fa5b 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -267,28 +267,54 @@ init_command = [ ] [[linter]] - code = 'PYUPGRADE' - is_formatter = true - include_patterns = [ - 'lintrunner_adapters/**/*.py', - 'lintrunner_adapters/**/*.pyi', - ] - exclude_patterns = [] - command = [ - 'python', - '-m', - 'lintrunner_adapters', - 'run', - 'pyupgrade_linter', - '--py37-plus', - '@{{PATHSFILE}}' - ] - init_command = [ - 'python', - '-m', - 'lintrunner_adapters', - 'run', - 'pip_init', - '--dry-run={{DRYRUN}}', - 'pyupgrade==3.3.1', - ] +code = 'PYUPGRADE' +is_formatter = true +include_patterns = [ + 'lintrunner_adapters/**/*.py', + 'lintrunner_adapters/**/*.pyi', +] +exclude_patterns = [] +command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'pyupgrade_linter', + '--py37-plus', + '@{{PATHSFILE}}' +] +init_command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'pip_init', + '--dry-run={{DRYRUN}}', + 'pyupgrade==3.3.1', +] + +[[linter]] +code = 'REFURB' +include_patterns = [ + 'lintrunner_adapters/**/*.py', + 'lintrunner_adapters/**/*.pyi', +] +exclude_patterns = [] +command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'refurb_linter', + '--', + '@{{PATHSFILE}}' +] +init_command = [ + 'python', + '-m', + 'lintrunner_adapters', + 'run', + 'pip_init', + '--dry-run={{DRYRUN}}', + 'refurb==1.10.0', +] diff --git a/lintrunner_adapters/adapters/refurb_linter.py b/lintrunner_adapters/adapters/refurb_linter.py index 3407ce2..b0ce065 100644 --- a/lintrunner_adapters/adapters/refurb_linter.py +++ b/lintrunner_adapters/adapters/refurb_linter.py @@ -45,10 +45,11 @@ def _test_results_re() -> None: def format_lint_message(message: str, code: str, show_disable: bool) -> str: - formatted = f"{message}\n" + formatted = f"{message}" if show_disable: formatted += textwrap.dedent( f""" + To disable, use [tool.refurb] ignore = [{code}] @@ -111,7 +112,8 @@ def check_files( def main() -> None: parser = argparse.ArgumentParser( - description=f"refurb wrapper linter. Linter code: {LINTER_CODE}", + description=f"refurb wrapper linter. Linter code: {LINTER_CODE}. " + f"Use pyproject.toml to configure any refurb settings.", fromfile_prefix_chars="@", ) parser.add_argument( diff --git a/pyproject.toml b/pyproject.toml index b85c6b2..f8ebfde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,3 +73,6 @@ disable = [ "unused-argument", "unused-import", ] + +[tool.refurb] +python_version = "3.7" From bbaf28e0f8a78c5d3d1b0b8fc953f071409f36f3 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 22 Jan 2023 23:37:13 +0300 Subject: [PATCH 5/5] Address review comments --- .lintrunner.toml | 2 +- examples/adapters/refurb/.lintrunner.toml | 2 +- lintrunner_adapters/adapters/flake8_linter.py | 6 +++--- lintrunner_adapters/adapters/newlines_linter.py | 6 +++--- pyproject.toml | 1 + 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index f58fa5b..fe2c86a 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -316,5 +316,5 @@ init_command = [ 'run', 'pip_init', '--dry-run={{DRYRUN}}', - 'refurb==1.10.0', + 'refurb==1.10.0;python_version>="3.10"', ] diff --git a/examples/adapters/refurb/.lintrunner.toml b/examples/adapters/refurb/.lintrunner.toml index 1e10b97..9f3c923 100644 --- a/examples/adapters/refurb/.lintrunner.toml +++ b/examples/adapters/refurb/.lintrunner.toml @@ -10,7 +10,7 @@ command = [ 'lintrunner_adapters', 'run', 'refurb_linter', - '--config=pyproject.toml', + '--config-file=pyproject.toml', '--show-disable', '--severity=FURB101:advice', '--severity=FURB102:warning', diff --git a/lintrunner_adapters/adapters/flake8_linter.py b/lintrunner_adapters/adapters/flake8_linter.py index 811b9b2..f2cef78 100644 --- a/lintrunner_adapters/adapters/flake8_linter.py +++ b/lintrunner_adapters/adapters/flake8_linter.py @@ -134,7 +134,7 @@ def get_issue_severity(code: str) -> LintSeverity: # "T49": internal type checker errors or unmatched messages if any( code.startswith(x) - for x in [ + for x in ( "B9", "C4", "C9", @@ -143,13 +143,13 @@ def get_issue_severity(code: str) -> LintSeverity: "E5", "T400", "T49", - ] + ) ): return LintSeverity.ADVICE # "F821": Undefined name # "E999": syntax error - if any(code.startswith(x) for x in ["F821", "E999"]): + if any(code.startswith(x) for x in ("F821", "E999")): return LintSeverity.ERROR # "F": PyFlakes Error diff --git a/lintrunner_adapters/adapters/newlines_linter.py b/lintrunner_adapters/adapters/newlines_linter.py index 16e8bb5..ee32997 100644 --- a/lintrunner_adapters/adapters/newlines_linter.py +++ b/lintrunner_adapters/adapters/newlines_linter.py @@ -21,11 +21,11 @@ def check_file(filename: str) -> LintMessage | None: with open(filename, "rb") as f: lines = f.readlines() - if len(lines) == 0: + if not lines: # File is empty, just leave it alone. return None - if len(lines) == 1 and len(lines[0]) == 1: + if len(lines) == len(lines[0]) == 1: # file is wrong whether or not the only byte is a newline return LintMessage( path=filename, @@ -77,7 +77,7 @@ def check_file(filename: str) -> LintMessage | None: for idx, line in enumerate(lines): if len(line) >= 2 and line[-1] == NEWLINE and line[-2] == CARRIAGE_RETURN: if not has_changes: - original_lines = list(lines) + original_lines = lines.copy() has_changes = True lines[idx] = line[:-2] + b"\n" diff --git a/pyproject.toml b/pyproject.toml index f8ebfde..d0dfb69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,3 +76,4 @@ disable = [ [tool.refurb] python_version = "3.7" +disable = ["FURB101", "FURB150"] # disable suggestions using pathlib