chore: speed up functional tests (#598)

* chore: speed up functional tests by installing `deptry` from wheel

* chore: run functional tests in parallel

* test: move arg types tests to unit

* test: typo wheel path

Co-authored-by: Florian Maas <fpgmaas@gmail.com>

* chore: single `test` command`

---------

Co-authored-by: Florian Maas <fpgmaas@gmail.com>

.github/workflows/main.yml

@@ -53,8 +53,11 @@ jobs:
       - name: Check typing
         run: pdm run mypy
 
-      - name: Run tests
-        run: pdm run pytest tests --cov --cov-config=pyproject.toml --cov-report=xml
+      - name: Run unit tests
+        run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml
+
+      - name: Run functional tests
+        run: pdm run pytest tests/functional -n auto --dist loadgroup
 
       - name: Upload coverage reports to Codecov with GitHub Action on Python 3.11 and x86_64
         uses: codecov/codecov-action@v4
@@ -79,8 +82,11 @@ jobs:
           target: ${{ matrix.target }}
           python-target: ${{ matrix.target }}
 
-      - name: Run tests
-        run: pdm run pytest tests --cov --cov-config=pyproject.toml --cov-report=xml
+      - name: Run unit tests
+        run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml
+
+      - name: Run functional tests
+        run: pdm run pytest tests/functional -n auto --dist loadgroup
 
   macos:
     runs-on: macos-latest
@@ -99,8 +105,11 @@ jobs:
           target: ${{ matrix.target }}
           python-target: ${{ matrix.target == 'aarch64' && 'arm64' || 'x64' }}
 
-      - name: Run tests
-        run: pdm run pytest tests --cov --cov-config=pyproject.toml --cov-report=xml
+      - name: Run unit tests
+        run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml
+
+      - name: Run functional tests
+        run: pdm run pytest tests/functional -n auto --dist loadgroup
 
   check-docs:
     runs-on: ubuntu-latest

Makefile

@@ -15,12 +15,20 @@ check: ## Run code quality tools.
 	@pdm run deptry python
 
 .PHONY: test
-test: ## Test the code with pytest.
-	@echo "🚀 Testing code: Running pytest"
-	@pdm run pytest --cov --cov-config=pyproject.toml --cov-report=xml
+test: test-unit test-functional
+
+.PHONY: test-unit
+test-unit: ## Run unit tests.
+	@echo "🚀 Running unit tests"
+	@pdm run pytest tests/unit
+
+.PHONY: test-functional
+test-functional: ## Run functional tests.
+	@echo "🚀 Running functional tests"
+	@pdm run pytest tests/functional -n auto --dist loadgroup
 
 .PHONY: build
-build: clean-build ## Build wheel and sdist files using PDM.
+build: ## Build wheel and sdist files using PDM.
 	@echo "🚀 Creating wheel and sdist files"
 	@maturin build
 

docs/contributing.md

@@ -79,7 +79,7 @@ Ready to contribute? Here's how to set up _deptry_ for local development. Please
 
 6. If you are adding a feature or fixing a bug, make sure to add tests in the `tests` directory.
 
-7. Once you're done, validate that all unit tests are passing:
+7. Once you're done, validate that all unit and functional tests are passing:
     ```bash
     make test
     ```

pyproject.toml

@@ -34,6 +34,7 @@ dev = [
     "pre-commit==3.5.0",
     "pytest==8.1.1",
     "pytest-cov==4.1.0",
+    "pytest-xdist[psutil]==3.5.0",
 ]
 docs = [
     "mkdocs==1.5.3",

tests/functional/cli/test_cli.py

@@ -4,17 +4,20 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
 from click.testing import CliRunner
 
 from deptry.cli import deptry
+from tests.functional.utils import Project
 from tests.utils import get_issues_report, stylize
 
 if TYPE_CHECKING:
     from tests.utils import PoetryVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_returns_error(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 
@@ -71,8 +74,9 @@ def test_cli_returns_error(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_ignore_notebooks(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . --ignore-notebooks -o {issue_report}")
 
@@ -141,22 +145,25 @@ def test_cli_ignore_notebooks(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_ignore_flags(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         result = virtual_env.run("deptry . --per-rule-ignores DEP002=isort|pkginfo|requests -im white -id black")
 
         assert result.returncode == 0
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_ignore_flag(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         result = virtual_env.run("deptry . --ignore DEP001,DEP002,DEP003,DEP004")
 
         assert result.returncode == 0
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_exclude(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . --exclude src/notebook.ipynb -o {issue_report}")
 
@@ -225,8 +232,9 @@ def test_cli_exclude(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_extend_exclude(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -ee src/notebook.ipynb -o {issue_report}")
 
@@ -295,8 +303,9 @@ def test_cli_extend_exclude(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_known_first_party(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . --known-first-party white -o {issue_report}")
 
@@ -341,8 +350,9 @@ def test_cli_known_first_party(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_not_verbose(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 
@@ -400,8 +410,9 @@ def test_cli_not_verbose(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_verbose(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . --verbose -o {issue_report}")
 
@@ -459,8 +470,9 @@ def test_cli_verbose(poetry_venv_factory: PoetryVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_with_no_ansi(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         result = virtual_env.run("deptry . --no-ansi")
 
         expected_output = [
@@ -480,8 +492,9 @@ def test_cli_with_no_ansi(poetry_venv_factory: PoetryVenvFactory) -> None:
         assert result.stderr == "\n".join(expected_output)
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_with_not_json_output(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         json_files_count = len(list(Path().glob("*.json")))
 
         result = virtual_env.run("deptry .")
@@ -521,8 +534,9 @@ def test_cli_with_not_json_output(poetry_venv_factory: PoetryVenvFactory) -> Non
         assert result.stderr == "\n".join(expected_output)
 
 
+@pytest.mark.xdist_group(name=Project.EXAMPLE)
 def test_cli_with_json_output(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("example_project") as virtual_env:
+    with poetry_venv_factory(Project.EXAMPLE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 

tests/functional/cli/test_cli_gitignore.py

@@ -4,14 +4,18 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PipVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.GITIGNORE)
 def test_cli_gitignore_is_used(pip_venv_factory: PipVenvFactory) -> None:
-    with pip_venv_factory("project_with_gitignore") as virtual_env:
+    with pip_venv_factory(Project.GITIGNORE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 
@@ -56,8 +60,9 @@ def test_cli_gitignore_is_used(pip_venv_factory: PipVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.GITIGNORE)
 def test_cli_gitignore_is_not_used(pip_venv_factory: PipVenvFactory) -> None:
-    with pip_venv_factory("project_with_gitignore") as virtual_env:
+    with pip_venv_factory(Project.GITIGNORE) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . --exclude build/|src/bar.py -o {issue_report}")
 

tests/functional/cli/test_cli_multiple_source_directories.py

@@ -4,14 +4,18 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PipVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.MULTIPLE_SOURCE_DIRECTORIES)
 def test_cli_with_multiple_source_directories(pip_venv_factory: PipVenvFactory) -> None:
-    with pip_venv_factory("project_with_multiple_source_directories") as virtual_env:
+    with pip_venv_factory(Project.MULTIPLE_SOURCE_DIRECTORIES) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry src worker -o {issue_report}")
 

tests/functional/cli/test_cli_pdm.py

@@ -4,14 +4,18 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PDMVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.PDM)
 def test_cli_with_pdm(pdm_venv_factory: PDMVenvFactory) -> None:
-    with pdm_venv_factory("project_with_pdm") as virtual_env:
+    with pdm_venv_factory(Project.PDM) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 

tests/functional/cli/test_cli_pep_621.py

@@ -4,14 +4,18 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PipVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.PEP_621)
 def test_cli_with_pep_621(pip_venv_factory: PipVenvFactory) -> None:
-    with pip_venv_factory("pep_621_project") as virtual_env:
+    with pip_venv_factory(Project.PEP_621) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 

tests/functional/cli/test_cli_poetry.py

@@ -4,14 +4,18 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PoetryVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.POETRY)
 def test_cli_with_poetry(poetry_venv_factory: PoetryVenvFactory) -> None:
-    with poetry_venv_factory("project_with_poetry") as virtual_env:
+    with poetry_venv_factory(Project.POETRY) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry . -o {issue_report}")
 

tests/functional/cli/test_cli_pyproject_different_directory.py

@@ -4,15 +4,19 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PipVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.PYPROJECT_DIFFERENT_DIRECTORY)
 def test_cli_with_pyproject_different_directory(pip_venv_factory: PipVenvFactory) -> None:
     with pip_venv_factory(
-        "project_with_pyproject_different_directory", install_command="pip install ./a_sub_directory"
+        Project.PYPROJECT_DIFFERENT_DIRECTORY, install_command="pip install ./a_sub_directory"
     ) as virtual_env:
         issue_report = f"{uuid.uuid4()}.json"
         result = virtual_env.run(f"deptry src --config a_sub_directory/pyproject.toml -o {issue_report}")

tests/functional/cli/test_cli_requirements_txt.py

@@ -4,15 +4,19 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pytest
+
+from tests.functional.utils import Project
 from tests.utils import get_issues_report
 
 if TYPE_CHECKING:
     from tests.utils import PipVenvFactory
 
 
+@pytest.mark.xdist_group(name=Project.REQUIREMENTS_TXT)
 def test_cli_single_requirements_txt(pip_venv_factory: PipVenvFactory) -> None:
     with pip_venv_factory(
-        "project_with_requirements_txt",
+        Project.REQUIREMENTS_TXT,
         install_command=(
             "pip install -r requirements.txt -r requirements-dev.txt -r requirements-2.txt -r requirements-typing.txt"
         ),
@@ -100,9 +104,10 @@ def test_cli_single_requirements_txt(pip_venv_factory: PipVenvFactory) -> None:
         ]
 
 
+@pytest.mark.xdist_group(name=Project.REQUIREMENTS_TXT)
 def test_cli_multiple_requirements_txt(pip_venv_factory: PipVenvFactory) -> None:
     with pip_venv_factory(
-        "project_with_requirements_txt",
+        Project.REQUIREMENTS_TXT,
         install_command=(
             "pip install -r requirements.txt -r requirements-dev.txt -r requirements-2.txt -r requirements-typing.txt"
         ),