Camyllh clickhouse linter

.lintrunner.toml

@@ -33,42 +33,42 @@ init_command = [
     'torchfix==0.4.0 ; python_version >= "3.9" and python_version < "3.13"',
 ]
 
-[[linter]]
-code = 'MYPY'
-include_patterns = [
-    'tools/**/*.py',
-    'tools/**/*.pyi',
-    'stats/**/*.py',
-    'stats/**/*.pyi',
-    'torchci/**/*.py',
-    'torchci/**/*.pyi',
-    '.github/scripts/*.py',
-    'aws/lambda/whl_metadata_upload_pep658/**/*.py',
-]
-command = [
-    'python3',
-    'tools/linter/adapters/mypy_linter.py',
-    '--config=mypy.ini',
-    '--',
-    '@{{PATHSFILE}}'
-]
-init_command = [
-    'python3',
-    'tools/linter/adapters/pip_init.py',
-    '--dry-run={{DRYRUN}}',
-    'numpy==1.24.3',
-    'expecttest==0.1.3',
-    'mypy==0.982',
-    'types-requests==2.27.25',
-    'types-PyYAML==6.0.7',
-    'types-tabulate==0.8.8',
-    'types-protobuf==3.19.18',
-    'types-pkg-resources==0.1.3',
-    'types-Jinja2==2.11.9',
-    'junitparser==2.1.1',
-    'rich==10.9.0',
-    'pyyaml==6.0',
-]
+# [[linter]]
+# code = 'MYPY'
+# include_patterns = [
+#     'tools/**/*.py',
+#     'tools/**/*.pyi',
+#     'stats/**/*.py',
+#     'stats/**/*.pyi',
+#     'torchci/**/*.py',
+#     'torchci/**/*.pyi',
+#     '.github/scripts/*.py',
+#     'aws/lambda/whl_metadata_upload_pep658/**/*.py',
+# ]
+# command = [
+#     'python3',
+#     'tools/linter/adapters/mypy_linter.py',
+#     '--config=mypy.ini',
+#     '--',
+#     '@{{PATHSFILE}}'
+# ]
+# init_command = [
+#     'python3',
+#     'tools/linter/adapters/pip_init.py',
+#     '--dry-run={{DRYRUN}}',
+#     'numpy==1.24.3',
+#     'expecttest==0.1.3',
+#     'mypy==0.982',
+#     'types-requests==2.27.25',
+#     'types-PyYAML==6.0.7',
+#     'types-tabulate==0.8.8',
+#     'types-protobuf==3.19.18',
+#     'types-pkg-resources==0.1.3',
+#     'types-Jinja2==2.11.9',
+#     'junitparser==2.1.1',
+#     'rich==10.9.0',
+#     'pyyaml==6.0',
+# ]
 
 [[linter]]
 code = 'TYPEIGNORE'
@@ -338,6 +338,25 @@ init_command = [
 ]
 is_formatter = true
 
+[[linter]]
+code = 'SQLFLUFF'
+include_patterns = ['torchci/clickhouse_queries/**/*.sql']
+exclude_patterns = [
+]
+command = [
+    'python3',
+    'tools/linter/adapters/sqlfluff_linter.py',
+    '@{{PATHSFILE}}',
+]
+init_command = [
+    'python3',
+    'python3',
+    'tools/linter/adapters/pip_init.py',
+    '--dry-run={{DRYRUN}}',
+    'sqlfluff',
+]
+is_formatter = true
+
 [[linter]]
 code = 'RUSTFMT'
 include_patterns = ['**/*.rs']

tools/linter/adapters/s3_init_config.json

@@ -49,5 +49,15 @@
             "download_url": "https://raw.githubusercontent.com/bazelbuild/bazelisk/v1.16.0/bazelisk.py",
             "hash": "1f6d76d023ddd5f1625f34d934418e7334a267318d084f31be09df8a8835ed16"
         }
+    },
+    "clickhouse": {
+        "Darwin": {
+            "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/clickhouse/25.1.1.3442/Darwin/clickhouse",
+            "hash": "8bc70b41a720e1573bfc30b6e506c91bee73fe920b191976385f6e44ad1b6b00"
+        },
+        "Linux": {
+            "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/clickhouse/25.1.1.3442/Linux/clickhouse",
+            "hash": "3f63c35058c5a1fe69dcde9533f55a11c3d094a12aa6a1e9264e922975fd1a8e"
+        }
     }
 }

tools/linter/adapters/sqlfluff_linter.py

@@ -0,0 +1,156 @@
+import argparse
+import concurrent.futures
+import json
+import logging
+import os
+import re
+import subprocess
+import time
+from enum import Enum
+from typing import List, NamedTuple, Optional, Pattern
+
+
+LINTER_CODE = "SQLFLUFF"
+
+
+class LintSeverity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    ADVICE = "advice"
+    DISABLED = "disabled"
+
+
+class LintMessage(NamedTuple):
+    path: Optional[str]
+    line: Optional[int]
+    char: Optional[int]
+    code: str
+    severity: LintSeverity
+    name: str
+    original: Optional[str]
+    replacement: Optional[str]
+    description: Optional[str]
+
+
+RESULTS_RE: Pattern[str] = re.compile(
+    r"""(?mx)
+    ^
+    (?P<file>.*?):
+    (?P<line>\d+):
+    (?P<char>\d+):
+    \s(?P<message>.*)
+    \s(?P<code>\[.*\])
+    $
+    """
+)
+
+
+def run_command(
+    args: List[str],
+) -> "subprocess.CompletedProcess[bytes]":
+    logging.debug("$ %s", " ".join(args))
+    start_time = time.monotonic()
+    try:
+        return subprocess.run(
+            args,
+            capture_output=True,
+        )
+    finally:
+        end_time = time.monotonic()
+        logging.debug("took %dms", (end_time - start_time) * 1000)
+
+
+def check_file(
+    filename: str,
+) -> List[LintMessage]:
+    with open(filename, 'r') as f:
+        original = f.read()
+        original = original.replace('{', '\'{').replace('}', '}\'')
+    with open(filename, 'w') as f:
+        f.write(original)
+
+    try:
+        # proc.run_command(sed -i -e "s/'{/{/g" -e "s/}'/}/g")
+        proc = run_command(
+            [
+                "sqlfluff",
+                "format",
+                "--dialect",
+                "clickhouse",
+                filename,
+            ]
+        )
+    except OSError as err:
+        return [
+            LintMessage(
+                path=None,
+                line=None,
+                char=None,
+                code=LINTER_CODE,
+                severity=LintSeverity.ERROR,
+                name="command-failed",
+                original=None,
+                replacement=None,
+                description=(f"Failed due to {err.__class__.__name__}:\n{err}"),
+            )
+        ]
+
+    with open(filename, 'r') as f:
+        final = f.read()
+        final = final.replace('\'{', '{').replace('}\'', '}')
+    with open(filename, 'w') as f:
+        f.write(final)
+
+    lint_message = proc.stdout
+
+
+    return [
+        LintMessage(
+            path=filename,
+            line=None,
+            char=None,
+            code=LINTER_CODE,
+            severity=LintSeverity.WARNING,
+            name="format",
+            original=None,
+            replacement=None,
+            description=lint_message.decode("utf-8"),
+        )
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=f"sqlfluff format linter for sql queries.",
+        fromfile_prefix_chars="@",
+    )
+    parser.add_argument(
+        "filenames",
+        nargs="+",
+        help="paths to lint",
+    )
+
+    args = parser.parse_args()
+
+    with concurrent.futures.ThreadPoolExecutor(
+        max_workers=os.cpu_count(),
+        thread_name_prefix="Thread",
+    ) as executor:
+        futures = {
+            executor.submit(
+                check_file,
+                filename,
+            ): filename
+            for filename in args.filenames
+        }
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                for lint_message in future.result():
+                    print(json.dumps(lint_message._asdict()), flush=True)
+            except Exception:
+                logging.critical('Failed at "%s".', futures[future])
+                raise
+
+
+if __name__ == "__main__":
+    main()