Refactor eco testing (#3400)

.github/workflows/tox.yml

@@ -84,6 +84,15 @@ jobs:
             ~/.cache/pre-commit
           key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }}
 
+      - name: Set eco cache
+        uses: actions/cache@v3
+        if: ${{ matrix.passed_name == 'eco' }}
+        with:
+          path: |
+            .cache/eco
+            ~/.cache/ansible-compat
+          key: eco-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('tools/test-eco.sh') }}
+
       - name: Set galaxy cache
         uses: actions/cache@v3
         if: ${{ startsWith(matrix.passed_name, 'py') }}

pyproject.toml

@@ -221,7 +221,6 @@ python_files = [
 # See: https://github.com/pytest-dev/pytest/issues/6451#issuecomment-687043537
 # testpaths =
 xfail_strict = true
-markers = ["eco: Tests effects on a set of 3rd party ansible repositories"]
 
 [tool.ruff]
 ignore = [

src/ansiblelint/app.py

@@ -183,7 +183,12 @@ def report_outcome(
         matched_rules = self._get_matched_skippable_rules(result.matches)
 
         if matched_rules and self.options.generate_ignore:
-            ignore_file_path = Path(IGNORE_FILE.default)
+            # ANSIBLE_LINT_IGNORE_FILE environment variable overrides default
+            # dumping location in linter and is not documented or supported. We
+            # use this only for testing purposes.
+            ignore_file_path = Path(
+                os.environ.get("ANSIBLE_LINT_IGNORE_FILE", IGNORE_FILE.default),
+            )
             console_stderr.print(f"Writing ignore file to {ignore_file_path}")
             lines = set()
             for rule in result.matches:

tools/test-eco.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eu
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [[ -d "${SCRIPT_DIR}/../.cache/eco/.git" ]]; then
+  git -C "${SCRIPT_DIR}/../.cache/eco" pull
+else
+  mkdir -p "${SCRIPT_DIR}/../.cache"
+  git clone --recursive https://github.com/ansible-community/ansible-lint-eco "${SCRIPT_DIR}/../.cache/eco"
+fi
+pushd "${SCRIPT_DIR}/../.cache/eco/projects" > /dev/null
+
+
+for i in $(ls -d */); do
+  DIR=${i%%/}
+  RC=0
+  pushd $DIR > /dev/null
+  # Calling ansible lint without any positional arguments inside repository root
+  SECONDS=0
+  ANSIBLE_LINT_IGNORE_FILE=../$DIR.ignore.txt ansible-lint -qq --generate-ignore -f codeclimate | python3 -m json.tool > ../$DIR.json ||
+  RC=$?
+  echo "Got $RC RC on $DIR in $SECONDS seconds"
+  popd > /dev/null
+done
+popd > /dev/null
+# Fail if git reports dirty at the end
+git diff --exit-code

tox.ini

@@ -168,12 +168,9 @@ deps =
 extras =
   test
 commands =
-  python3 -m venv .tox/venv
-  .tox/venv/bin/pip install -q git+https://github.com/ansible/ansible-lint@main
-  pytest -n auto --durations=3 -m eco
+  sh -c tools/test-eco.sh
 allowlist_externals =
   {[testenv]allowlist_externals}
-  .tox/venv/bin/pip
 setenv =
   PYTEST_REQPASS=7