From 1021e53cd5e6591a269deb615fbe24acf0140a3f Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Mon, 5 Aug 2024 09:18:34 +0100 Subject: [PATCH] Upgrade linters and refactor how they run (#221) --- .config/dictionary.txt | 1 + .github/workflows/ack.yml | 7 ++- .github/workflows/linters.yml | 43 ------------------- .isort.cfg | 2 - .pre-commit-config.yaml | 43 ++++++++++++++++++- .../event_filter/insert_hosts_to_meta.py | 1 + .../eda/plugins/event_source/journald.py | 6 ++- .../eda/plugins/event_source/webhook.py | 3 ++ lint_requirements.txt | 2 - pyproject.toml | 22 ++++++++++ test_requirements.txt | 1 - tox.ini | 31 ++----------- 12 files changed, 83 insertions(+), 79 deletions(-) delete mode 100644 .github/workflows/linters.yml delete mode 100644 .isort.cfg delete mode 100644 lint_requirements.txt create mode 100644 pyproject.toml diff --git a/.config/dictionary.txt b/.config/dictionary.txt index 29ae3b69..632253ec 100644 --- a/.config/dictionary.txt +++ b/.config/dictionary.txt @@ -33,6 +33,7 @@ rulebook rulebooks ruleset servicebus +skipsdist snakeoil storepass testenv diff --git a/.github/workflows/ack.yml b/.github/workflows/ack.yml index 60853af3..1a559ab3 100644 --- a/.github/workflows/ack.yml +++ b/.github/workflows/ack.yml @@ -1,10 +1,15 @@ --- # See https://github.com/ansible/devtools/blob/main/.github/workflows/ack.yml name: ack -"on": +on: pull_request_target: types: [opened, labeled, unlabeled, synchronize] +permissions: + checks: write + contents: write # needed to update release + pull-requests: write # pr approval and merge + jobs: ack: uses: ansible/team-devtools/.github/workflows/ack.yml@main diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml deleted file mode 100644 index 760a9c5b..00000000 --- a/.github/workflows/linters.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: Run linters -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Run the linters tests every 8 hours. - # This will help to identify faster if - # there is a CI failure related to a - # change in any dependency. - schedule: - - cron: '0 */8 * * *' -jobs: - black: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - - - name: Run black - uses: psf/black@stable - isort: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - - - name: Run isort - uses: isort/isort-action@v1.1.0 - flake8: - runs-on: ubuntu-latest - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - - - name: Install flake8 - run: | - python3 -m pip install -U pip - python3 -m pip install -U flake8 - - - name: Run flake8 - run: flake8 diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index b9fb3f3e..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c7761fe..606ee11d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,15 +6,54 @@ ci: exclude: "^.*\\.md$" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer language_version: python3 - repo: https://github.com/streetsidesoftware/cspell-cli - rev: v8.13.0 + rev: v8.13.1 hooks: - id: cspell + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + language_version: python3 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.5.6" + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/pylint + rev: v3.2.6 + hooks: + - id: pylint + args: + - --output-format=colorized + additional_dependencies: + - aiobotocore + - aiohttp + - aiokafka + - asyncmock + - azure-servicebus + - dpath + - kafka-python + - psycopg + - pytest + - pyyaml + - requests + - watchdog + - xxhash + - ansible-core - repo: local hooks: - id: ansible-test-sanity diff --git a/extensions/eda/plugins/event_filter/insert_hosts_to_meta.py b/extensions/eda/plugins/event_filter/insert_hosts_to_meta.py index 49aa03c0..d6ce65ff 100644 --- a/extensions/eda/plugins/event_filter/insert_hosts_to_meta.py +++ b/extensions/eda/plugins/event_filter/insert_hosts_to_meta.py @@ -49,6 +49,7 @@ class PathNotExistError(Exception): """Cannot find the path in the event.""" +# pylint: disable=too-many-arguments def main( event: dict[str, Any], host_path: str | None = None, diff --git a/extensions/eda/plugins/event_source/journald.py b/extensions/eda/plugins/event_source/journald.py index be3b2486..969c6f26 100644 --- a/extensions/eda/plugins/event_source/journald.py +++ b/extensions/eda/plugins/event_source/journald.py @@ -26,7 +26,11 @@ import asyncio from typing import Any -from systemd import journal +# https://github.com/pylint-dev/pylint/issues/7240 +# Also systemd-python fails to install on pre-commit.ci due to: +# No such file or directory: 'pkg-config' +# pylint: disable=import-error +from systemd import journal # type: ignore async def main(queue: asyncio.Queue, args: dict[str, Any]) -> str: # noqa: D417 diff --git a/extensions/eda/plugins/event_source/webhook.py b/extensions/eda/plugins/event_source/webhook.py index 730f3d4b..4e3478e1 100644 --- a/extensions/eda/plugins/event_source/webhook.py +++ b/extensions/eda/plugins/event_source/webhook.py @@ -109,6 +109,9 @@ async def _hmac_verify(request: web.Request) -> bool: event_digest = base64.b64encode(event_hmac.digest()).decode("utf-8") elif hmac_format == "hex": event_digest = event_hmac.hexdigest() + else: + msg = f"Unsupported HMAC header format {hmac_format}" + raise ValueError(msg) return hmac.compare_digest(hmac_header_digest, event_digest) diff --git a/lint_requirements.txt b/lint_requirements.txt deleted file mode 100644 index 699844e3..00000000 --- a/lint_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ruff==0.0.285 -pylint==2.17.5 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2579c997 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.black] +target-version = ["py310"] + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 + +[tool.pylint.MASTER] +# Temporary ignore until we are able to address issues on these: +ignore-paths = "^(demos/dynatrace-demo/fake_app.py|tests/|plugins/modules).*$" + +[tool.pylint."MESSAGES CONTROL"] +disable = [ + "duplicate-code", + "pointless-string-statement", + "too-few-public-methods", +] diff --git a/test_requirements.txt b/test_requirements.txt index 6386d87f..2f3d8daf 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,4 @@ -r requirements.txt --r lint_requirements.txt pytest asyncmock pytest-asyncio diff --git a/tox.ini b/tox.ini index 4bd47735..d129be9a 100644 --- a/tox.ini +++ b/tox.ini @@ -3,43 +3,20 @@ # {posargs} in this case would be the path to collection root relative from the .github/workflows dir (`../..`) [tox] -# envlist = ruff, darglint, pylint-event-source, pylint-event-filter -envlist = lint, ruff, pylint-event-source, pylint-event-filter, isort, black +envlist = lint requires = lint - ruff darglint - pylint +skipsdist = true # this repo is not a python package [testenv] basepython = python3.9, python3.10 + [testenv:lint] deps = pre-commit -desc = Run linters +description = Run linters commands = pre-commit run -a -[testenv:ruff] -deps = -r{toxinidir}/lint_requirements.txt -commands = ruff check --select ALL --ignore INP001,RUF100,FA102,PLR0913 -q extensions/eda/plugins - [testenv:darglint] deps = darglint commands = darglint -s numpy -z full extensions/eda/plugins - -[testenv:isort] -deps = isort -commands = isort {toxinidir} - -[testenv:black] -deps = black -commands = black {toxinidir} - -[testenv:pylint-event-source] -deps = -r{toxinidir}/lint_requirements.txt - -r{toxinidir}/requirements.txt -commands = pylint extensions/eda/plugins/event_source/*.py --output-format=parseable -sn --disable R0801,R0903,W0105 - -[testenv:pylint-event-filter] -deps = -r{toxinidir}/lint_requirements.txt - -r{toxinidir}/requirements.txt -commands = pylint extensions/eda/plugins/event_filter/*.py --output-format=parseable -sn --disable R0801