Tools for speeding up tests using Git

CHANGES.rst

@@ -5,6 +5,10 @@ These features will be included in the next release:
 
 Added
 -----
+- Tools for easy creation of differently scoped Git repository fixtures for tests.
+  Helps speed up parameterized tests that need a Git repository, especially on Windows
+  where Git process forks are comically expensive. The ``test_git.py`` test module now
+  makes use of this and runs in 9s instead of 18s on one Windows laptop.
 - Unit tests of configuration file options for `darkgraylib.config.load_config`.
 
 Fixed

pyproject.toml

@@ -41,6 +41,7 @@ target-version = "py38"
 select = ["ALL"]
 ignore = [
     "ANN101",  # Missing type annotation for `self` in method
+    "COM812",  # Trailing comma missing
     "D203",  # One blank line required before class docstring
     "D213",  # Multi-line docstring summary should start at the second line
     "D400",  # First line should end with a period (duplicates D415)
@@ -51,6 +52,7 @@ ignore = [
     "ANN001",  # Missing type annotation for function argument
     "ANN201",  # Missing return type annotation for public function
     "ANN204",  # Missing return type annotation for special method `__init__`
+    "INP001",  # File is part of an implicit namespace package. Add an `__init__.py`.
     "C408",  # Unnecessary `dict` call (rewrite as a literal)
     "S101",  # Use of `assert` detected
 ]

setup.cfg

@@ -48,6 +48,8 @@ pygments.lexers =
 pytest11 =
     pytest_git_repo = darkgraylib.testtools.git_repo_plugin
     pytest_clear_black_cache = darkgraylib.testtools.clear_black_cache_plugin
+    pytest_temp_copy = darkgraylib.testtools.temp_copy
+    pytest_patching = darkgraylib.testtools.patching
 
 [options.extras_require]
 color =

src/darkgraylib/tests/test_git.py

@@ -1,15 +1,22 @@
+"""Tests for the `darkgraylib.git` module."""
+
+# pylint: disable=no-member  # context managers misfire Pylint's member-checking
+# pylint: disable=redefined-outer-name  # fixtures misfire Pylint's redefinition checks
+# pylint: disable=use-dict-literal  # dict() ok with kwparametrize
+
 import os
 import re
 from datetime import datetime, timedelta
 from pathlib import Path
 from subprocess import PIPE, CalledProcessError  # nosec
+from types import SimpleNamespace
 from typing import List, Union
 from unittest.mock import ANY, Mock, call, patch
 
 import pytest
 
 from darkgraylib import git
-from darkgraylib.testtools.git_repo_plugin import GitRepoFixture
+from darkgraylib.testtools.git_repo_plugin import GitRepoFixture, branched_repo
 from darkgraylib.testtools.helpers import raises_or_matches
 from darkgraylib.utils import GIT_DATEFORMAT, TextDocument
 
@@ -73,6 +80,17 @@ def test_git_get_mtime_at_commit():
         assert result == "2020-12-27 21:33:59.000000 +0000"
 
 
+@pytest.fixture(scope="module")
+def git_get_content_at_revision_repo(request, tmp_path_factory):
+    """Return Git repository fixture with a file that changes over time."""
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        repo.add({"my.txt": "original content"}, commit="Initial commit")
+        paths = repo.add({"my.txt": "modified content"}, commit="Second commit")
+        paths["my.txt"].write_bytes(b"new content")
+        os.utime(paths["my.txt"], (1000000000, 1000000000))
+        yield repo
+
+
 @pytest.mark.kwparametrize(
     dict(
         revision=":WORKTREE:",
@@ -91,15 +109,12 @@ def test_git_get_mtime_at_commit():
     ),
     dict(revision="HEAD~2", expect_lines=(), expect_mtime=False),
 )
-def test_git_get_content_at_revision(git_repo, revision, expect_lines, expect_mtime):
-    """darkgraylib.git.git_get_content_at_revision()"""
-    git_repo.add({"my.txt": "original content"}, commit="Initial commit")
-    paths = git_repo.add({"my.txt": "modified content"}, commit="Initial commit")
-    paths["my.txt"].write_bytes(b"new content")
-    os.utime(paths["my.txt"], (1000000000, 1000000000))
-
+def test_git_get_content_at_revision(
+    git_get_content_at_revision_repo, revision, expect_lines, expect_mtime
+):
+    """Test for `git.git_get_content_at_revision`."""
     result = git.git_get_content_at_revision(
-        Path("my.txt"), revision, cwd=Path(git_repo.root)
+        Path("my.txt"), revision, cwd=Path(git_get_content_at_revision_repo.root)
     )
 
     assert result.lines == expect_lines
@@ -177,6 +192,9 @@ def test_git_get_content_at_revision_obtain_file_content(
         assert text_document_class.method_calls == expect_textdocument_calls
 
 
+git_check_output_lines_repo = pytest.fixture(scope="module")(branched_repo)
+
+
 @pytest.mark.kwparametrize(
     dict(cmd=[], exit_on_error=True, expect_template=CalledProcessError(1, "")),
     dict(
@@ -227,8 +245,11 @@ def test_git_get_content_at_revision_obtain_file_content(
         expect_template=CalledProcessError(128, ""),
     ),
 )
-def test_git_check_output_lines(branched_repo, cmd, exit_on_error, expect_template):
+def test_git_check_output_lines(
+    git_check_output_lines_repo, cmd, exit_on_error, expect_template
+):
     """Unit test for :func:`git_check_output_lines`"""
+    branched_repo = git_check_output_lines_repo
     if isinstance(expect_template, BaseException):
         expect: Union[List[str], BaseException] = expect_template
     else:
@@ -238,6 +259,18 @@ def test_git_check_output_lines(branched_repo, cmd, exit_on_error, expect_templa
         check(git.git_check_output_lines(cmd, branched_repo.root, exit_on_error))
 
 
+@pytest.fixture(scope="module")
+def two_file_repo(request, tmp_path_factory):
+    """Make a Git repo with two files in the root, and the hash of the first commit."""
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        fixture = SimpleNamespace()
+        repo.add({"file1": "file1"}, commit="Initial commit")
+        fixture.initial = repo.get_hash()[:7]
+        repo.add({"file2": "file2"}, commit="Second commit")
+        fixture.root = repo.root
+        yield fixture
+
+
 @pytest.mark.kwparametrize(
     dict(
         cmd=["show", "{initial}:/.file2"],
@@ -268,32 +301,32 @@ def test_git_check_output_lines(branched_repo, cmd, exit_on_error, expect_templa
     expect_log=r"$",
 )
 def test_git_check_output_lines_stderr_and_log(
-    git_repo, capfd, caplog, cmd, exit_on_error, expect_exc, expect_stderr, expect_log
+    two_file_repo,
+    capfd,
+    caplog,
+    cmd,
+    exit_on_error,
+    expect_exc,
+    expect_stderr,
+    expect_log,
 ):
     """Git non-existing file error is logged and suppressed from stderr"""
-    git_repo.add({"file1": "file1"}, commit="Initial commit")
-    initial = git_repo.get_hash()[:7]
-    git_repo.add({"file2": "file2"}, commit="Second commit")
-    capfd.readouterr()  # flush captured stdout and stderr
-    cmdline = [s.format(initial=initial) for s in cmd]
+    cmdline = [s.format(initial=two_file_repo.initial) for s in cmd]
     with pytest.raises(expect_exc):
-        git.git_check_output_lines(cmdline, git_repo.root, exit_on_error)
+        git.git_check_output_lines(cmdline, two_file_repo.root, exit_on_error)
 
     outerr = capfd.readouterr()
     assert outerr.out == ""
     assert outerr.err == expect_stderr
-    expect_log_re = expect_log.format(initial=initial)
+    expect_log_re = expect_log.format(initial=two_file_repo.initial)
     assert re.search(expect_log_re, caplog.text), repr(caplog.text)
 
 
-def test_git_get_content_at_revision_stderr(git_repo, capfd, caplog):
-    """No stderr or log output from ``git_get_content_at_revision`` for missing file"""
-    git_repo.add({"file1": "file1"}, commit="Initial commit")
-    initial = git_repo.get_hash()[:7]
-    git_repo.add({"file2": "file2"}, commit="Second commit")
-    capfd.readouterr()  # flush captured stdout and stderr
-
-    result = git.git_get_content_at_revision(Path("file2"), initial, git_repo.root)
+def test_git_get_content_at_revision_stderr(two_file_repo, capfd, caplog):
+    """No stderr/log output from `git.git_get_content_at_revision` for missing file."""
+    result = git.git_get_content_at_revision(
+        Path("file2"), two_file_repo.initial, two_file_repo.root
+    )
 
     assert result == TextDocument()
     outerr = capfd.readouterr()
@@ -356,53 +389,99 @@ def test_git_get_content_at_revision_encoding(encodings_repo, commit, encoding,
     assert result.lines == lines
 
 
+@pytest.fixture(scope="module")
+def git_clone_local_branch_repo(request, tmp_path_factory):
+    """Git repository with three branches and a file with different content in each."""
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        repo.add({"a.py": "first"}, commit="first")
+        repo.create_branch("first", "HEAD")
+        repo.create_branch("second", "HEAD")
+        repo.add({"a.py": "second"}, commit="second")
+        repo.create_branch("third", "HEAD")
+        repo.add({"a.py": "third"}, commit="third")
+        yield repo
+
+
 @pytest.mark.kwparametrize(
     dict(branch="first", expect="first"),
     dict(branch="second", expect="second"),
     dict(branch="third", expect="third"),
     dict(branch="HEAD", expect="third"),
 )
-def test_git_clone_local_branch(git_repo, tmp_path, branch, expect):
-    """``git_clone_local()`` checks out the specified branch"""
-    git_repo.add({"a.py": "first"}, commit="first")
-    git_repo.create_branch("first", "HEAD")
-    git_repo.create_branch("second", "HEAD")
-    git_repo.add({"a.py": "second"}, commit="second")
-    git_repo.create_branch("third", "HEAD")
-    git_repo.add({"a.py": "third"}, commit="third")
-
-    with git.git_clone_local(git_repo.root, branch, tmp_path / "clone") as clone:
+def test_git_clone_local_branch(git_clone_local_branch_repo, tmp_path, branch, expect):
+    """`git_clone_local` checks out the specified branch."""
+    repo = git_clone_local_branch_repo
+    with git.git_clone_local(repo.root, branch, tmp_path / "clone") as clone:
         assert (clone / "a.py").read_text() == expect
 
 
+@pytest.fixture(scope="module")
+def git_clone_local_command_fixture(request, tmp_path_factory):
+    """Repository and other fixtures for `git.git_clone_local` tests."""
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        fixture = SimpleNamespace()
+        fixture.root = repo.root
+        repo.add({"a.py": "first"}, commit="first")
+        repo.create_branch("mybranch", "HEAD")
+        fixture.check_output = Mock(
+            wraps=git.check_output  # type: ignore[attr-defined]
+        )
+        temporary_path = tmp_path_factory.mktemp("git_clone_local_command")
+        fixture.clone = temporary_path / "clone"
+        fixture.check_output_opts = dict(
+            cwd=str(repo.root), encoding=None, stderr=PIPE, env=ANY
+        )
+        fixture.post_call = call(
+            ["git", "worktree", "remove", "--force", "--force", str(fixture.clone)],
+            **fixture.check_output_opts,
+        )
+        yield fixture
+
+
 @pytest.mark.kwparametrize(
     dict(branch="HEAD"),
     dict(branch="mybranch"),
 )
-def test_git_clone_local_command(git_repo, tmp_path, branch):
-    """``git_clone_local()`` issues the correct Git command and options"""
-    git_repo.add({"a.py": "first"}, commit="first")
-    git_repo.create_branch("mybranch", "HEAD")
-    check_output = Mock(wraps=git.check_output)  # type: ignore[attr-defined]
-    clone = tmp_path / "clone"
-    check_output_opts = dict(
-        cwd=str(git_repo.root), encoding=None, stderr=PIPE, env=ANY
-    )
+def test_git_clone_local_command(git_clone_local_command_fixture, branch):
+    """`git.git_clone_local` issues the correct Git command and options."""
+    fixture = git_clone_local_command_fixture
     pre_call = call(
-        ["git", "worktree", "add", "--quiet", "--force", "--force", str(clone), branch],
-        **check_output_opts,
-    )
-    post_call = call(
-        ["git", "worktree", "remove", "--force", "--force", str(clone)],
-        **check_output_opts,
+        [
+            "git",
+            "worktree",
+            "add",
+            "--quiet",
+            "--force",
+            "--force",
+            str(fixture.clone),
+            branch,
+        ],
+        **fixture.check_output_opts,
     )
-    with patch.object(git, "check_output", check_output):
-        with git.git_clone_local(git_repo.root, branch, clone) as result:
-            assert result == clone
+    with patch.object(git, "check_output", fixture.check_output), git.git_clone_local(
+        fixture.root, branch, fixture.clone
+    ) as result:
+        # function called, begin assertions
+
+        assert result == fixture.clone
+        fixture.check_output.assert_has_calls([pre_call])
+        fixture.check_output.reset_mock()
+    fixture.check_output.assert_has_calls([fixture.post_call])
+
 
-            check_output.assert_has_calls([pre_call])
-            check_output.reset_mock()
-    check_output.assert_has_calls([post_call])
+@pytest.fixture(scope="module")
+def git_get_root_repo(request, tmp_path_factory):
+    """Make a Git repository with files in the root and in subdirectories."""
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        repo.add(
+            {
+                "root.py": "root",
+                "subdir/sub.py": "sub",
+                "subdir/subsubdir/subsub.py": "subsub",
+            },
+            commit="Initial commit",
+        )
+        yield repo
 
 
 @pytest.mark.parametrize(
@@ -416,20 +495,11 @@ def test_git_clone_local_command(git_repo, tmp_path, branch):
         "subdir/subsubdir/subsub.py",
     ],
 )
-def test_git_get_root(git_repo, path):
-    """``git_get_root()`` returns repository root for any file or directory inside"""
-    git_repo.add(
-        {
-            "root.py": "root",
-            "subdir/sub.py": "sub",
-            "subdir/subsubdir/subsub.py": "subsub",
-        },
-        commit="Initial commit",
-    )
-
-    root = git.git_get_root(git_repo.root / path)
+def test_git_get_root(git_get_root_repo, path):
+    """`git.git_get_root` returns repository root for any file or directory inside."""
+    root = git.git_get_root(git_get_root_repo.root / path)
 
-    assert root == git_repo.root
+    assert root == git_get_root_repo.root
 
 
 @pytest.mark.parametrize(

src/darkgraylib/testtools/git_repo_plugin.py

@@ -1,25 +1,74 @@
 """Git repository fixture as a Pytest plugin"""
 
+# pylint: disable=no-member  # context managers misfire Pylint's member-checking
+
+from __future__ import annotations
+
 import os
 import re
+from contextlib import contextmanager
 from pathlib import Path
+from shutil import rmtree
 from subprocess import check_call  # nosec
-from typing import Dict, Iterable, List, Union
+from typing import Generator, Iterable
 
 import pytest
+from _pytest.tmpdir import _mk_tmp
 
 from darkgraylib.git import git_check_output_lines, git_get_version
 
 
 class GitRepoFixture:
     """Fixture for managing temporary Git repositories"""
 
-    def __init__(self, root: Path, env: Dict[str, str]):
+    def __init__(self, root: Path, env: dict[str, str]) -> None:
+        """Use given environment, and directory as the root of the Git repository."""
         self.root = root
         self.env = env
 
     @classmethod
-    def create_repository(cls, root: Path) -> "GitRepoFixture":
+    def tmp_repo(
+        cls, request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory
+    ) -> Generator[GitRepoFixture]:
+        """Create temporary Git repository and change current working directory into it.
+
+        This raw function needs to be turned into a fixture using `pytest.fixture` or a
+        context manager using `contextmanager`.
+        Examples::
+
+            git_repo = pytest.fixture(scope="module")(GitRepoFixture._tmp_repo)
+            def test_something(git_repo):
+                assert git_repo.root.is_dir()
+
+            my_fixture = contextmanager(GitRepoFixture._tmp_repo)
+            def test_something_else(request, tmp_path_factory):
+                with my_fixture(request, tmp_path_factory) as git_repo:
+                    assert git_repo.root.is_dir()
+
+        """
+        path = _mk_tmp(request, tmp_path_factory)
+        try:
+            with pytest.MonkeyPatch.context() as mp:
+                repository = cls.create_repository(path)
+                mp.chdir(repository.root)
+                # While `GitRepoFixture.create_repository()` already deletes `GIT_*`
+                # environment variables for any Git commands run by the fixture, let's
+                # explicitly remove `GIT_DIR` in case a test should call Git directly:
+                mp.delenv("GIT_DIR", raising=False)
+                yield repository
+        finally:
+            rmtree(path, ignore_errors=True)
+
+    @classmethod
+    @contextmanager
+    def context(
+        cls, request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory
+    ) -> Generator[GitRepoFixture]:
+        """Return a context manager for creating a temporary Git repository."""
+        yield from cls.tmp_repo(request, tmp_path_factory)
+
+    @classmethod
+    def create_repository(cls, root: Path) -> GitRepoFixture:
         """Fixture method for creating a Git repository in the given directory"""
         # For testing, ignore ~/.gitconfig settings like templateDir and defaultBranch.
         # Also, this makes sure GIT_DIR or other GIT_* variables are not set, and that
@@ -44,8 +93,10 @@ def _run_and_get_first_line(self, *args: str) -> str:
         return git_check_output_lines(list(args), Path(self.root))[0]
 
     def add(
-        self, paths_and_contents: Dict[str, Union[str, bytes, None]], commit: str = None
-    ) -> Dict[str, Path]:
+        self,
+        paths_and_contents: dict[str, str | bytes | None],
+        commit: str | None = None,
+    ) -> dict[str, Path]:
         """Add/remove/modify files and optionally commit the changes
 
         :param paths_and_contents: Paths of the files relative to repository root, and
@@ -100,7 +151,7 @@ def create_branch(self, new_branch: str, start_point: str) -> None:
         """Fixture method to create and check out new branch at given starting point"""
         self._run("checkout", "-b", new_branch, start_point)
 
-    def expand_root(self, lines: Iterable[str]) -> List[str]:
+    def expand_root(self, lines: Iterable[str]) -> list[str]:
         """Replace "{root/<path>}" in strings with the path in the temporary Git repo
 
         This is used to generate expected strings corresponding to locations of files in
@@ -120,22 +171,16 @@ def expand_root(self, lines: Iterable[str]) -> List[str]:
         ]
 
 
-@pytest.fixture
-def git_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> GitRepoFixture:
-    """Create a temporary Git repository and change current working directory into it"""
-    repository = GitRepoFixture.create_repository(tmp_path)
-    monkeypatch.chdir(tmp_path)
-    # While `GitRepoFixture.create_repository()` already deletes `GIT_*` environment
-    # variables for any Git commands run by the fixture, let's explicitly remove
-    # `GIT_DIR` in case a test should call Git directly:
-    monkeypatch.delenv("GIT_DIR", raising=False)
+git_repo = pytest.fixture(GitRepoFixture.tmp_repo)
+git_repo_m = pytest.fixture(scope="module")(GitRepoFixture.tmp_repo)
 
-    return repository
 
+def branched_repo(
+    request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory
+) -> Generator[GitRepoFixture]:
+    """Create an example Git repository with a master branch and a feature branch.
 
-@pytest.fixture(scope="module")
-def branched_repo(tmp_path_factory: pytest.TempPathFactory) -> GitRepoFixture:
-    """Create an example Git repository with a master branch and a feature branch
+    This raw function needs to be turned into a fixture using `pytest.fixture`.
 
     The history created is::
 
@@ -147,47 +192,46 @@ def branched_repo(tmp_path_factory: pytest.TempPathFactory) -> GitRepoFixture:
         * Initial commit
 
     """
-    tmpdir = tmp_path_factory.mktemp("branched_repo")
-    repo = GitRepoFixture.create_repository(tmpdir)
-    repo.add(
-        {
-            "del_master.py": "original",
-            "del_branch.py": "original",
-            "del_index.py": "original",
-            "del_worktree.py": "original",
-            "mod_master.py": "original",
-            "mod_branch.py": "original",
-            "mod_both.py": "original",
-            "mod_same.py": "original",
-            "keep.py": "original",
-        },
-        commit="Initial commit",
-    )
-    branch_point = repo.get_hash()
-    repo.add(
-        {
-            "del_master.py": None,
-            "add_master.py": "master",
-            "mod_master.py": "master",
-            "mod_both.py": "master",
-            "mod_same.py": "same",
-        },
-        commit="master",
-    )
-    repo.create_branch("branch", branch_point)
-    repo.add(
-        {
-            "del_branch.py": None,
-            "mod_branch.py": "branch",
-            "mod_both.py": "branch",
-            "mod_same.py": "same",
-        },
-        commit="branch",
-    )
-    repo.add(
-        {"del_index.py": None, "add_index.py": "index", "mod_index.py": "index"}
-    )
-    (repo.root / "del_worktree.py").unlink()
-    (repo.root / "add_worktree.py").write_bytes(b"worktree")
-    (repo.root / "mod_worktree.py").write_bytes(b"worktree")
-    return repo
+    with GitRepoFixture.context(request, tmp_path_factory) as repo:
+        repo.add(
+            {
+                "del_master.py": "original",
+                "del_branch.py": "original",
+                "del_index.py": "original",
+                "del_worktree.py": "original",
+                "mod_master.py": "original",
+                "mod_branch.py": "original",
+                "mod_both.py": "original",
+                "mod_same.py": "original",
+                "keep.py": "original",
+            },
+            commit="Initial commit",
+        )
+        branch_point = repo.get_hash()
+        repo.add(
+            {
+                "del_master.py": None,
+                "add_master.py": "master",
+                "mod_master.py": "master",
+                "mod_both.py": "master",
+                "mod_same.py": "same",
+            },
+            commit="master",
+        )
+        repo.create_branch("branch", branch_point)
+        repo.add(
+            {
+                "del_branch.py": None,
+                "mod_branch.py": "branch",
+                "mod_both.py": "branch",
+                "mod_same.py": "same",
+            },
+            commit="branch",
+        )
+        repo.add(
+            {"del_index.py": None, "add_index.py": "index", "mod_index.py": "index"}
+        )
+        (repo.root / "del_worktree.py").unlink()
+        (repo.root / "add_worktree.py").write_bytes(b"worktree")
+        (repo.root / "mod_worktree.py").write_bytes(b"worktree")
+        yield repo

src/darkgraylib/testtools/patching.py

@@ -0,0 +1,17 @@
+"""Helpers for patching in tests."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from typing import Iterator
+
+
+@pytest.fixture(scope="module")
+def monkeymodule() -> Iterator[pytest.MonkeyPatch]:
+    """Return a module-scope monkeypatch fixture."""
+    with pytest.MonkeyPatch.context() as monkey_patch:
+        yield monkey_patch

src/darkgraylib/testtools/temp_copy.py

@@ -0,0 +1,31 @@
+"""Pytest fixture factory for making temporary copies of directory trees."""
+
+from __future__ import annotations
+
+import re
+from contextlib import contextmanager
+from shutil import copytree
+from typing import TYPE_CHECKING, Callable, ContextManager
+
+import pytest
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from typing import Generator
+
+
+@pytest.fixture
+def make_temp_copy(
+    request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory
+) -> Callable[[Path], ContextManager[Path]]:
+    """Pytest fixture to create a temporary clone of a directory structure."""
+
+    @contextmanager
+    def temp_copy_factory(path: Path) -> Generator[Path]:
+        max_len = 30
+        name = re.sub(r"\W", "_", f"clone_{request.node.name}")[:max_len]
+        clone = tmp_path_factory.mktemp(name, numbered=True) / path.name
+        copytree(path, clone)
+        yield clone
+
+    return temp_copy_factory