Support Windows for pre-commit hook usage (#165)

Remove any OS-specific commands in order to support Windows.

Several CI test jobs have been added to run all pre-commit hooks
in both Linux and Windows environments, furthermore all pytest
jobs have been extended to include Windows environments.

The expanded test-space have revealed some weaknesses regarding
path-handling, which have been fixed through the usage of
pathlib.PurePosixPath. This ensures the user inputs should still match
what is explained in the documentation for paths, namely that one
should use POSIX (Linux) type path strings (with forward slashes (/)
and excluding drives and similar).

Finally, any Linux-specific commands used through invoke.context.run(),
which calls the OS shell, have been removed in favor of a Pythonic
approach. Note: It is still necessary that the git command can be run in
the given shell where pre-commit is run.

Ensure changes from testing `ci-cd setver` hooks is OK:
Change target version to 0.0.0, which should never be an actual version
used, meaning there is no single point this test job should suddenly not
incur a "change" in the base files.

.github/utils/.pre-commit-config_testing.yaml

@@ -0,0 +1,34 @@
+repos:
+  - repo: .
+    rev: HEAD
+    hooks:
+    - id: docs-api-reference
+      args:
+      - "--package-dir=ci_cd"
+      - "--debug"
+    - id: docs-landing-page
+    - id: update-pyproject
+
+  - repo: local
+    hooks:
+    - id: set-version
+      name: Set package version
+      entry: "ci-cd setver"
+      language: python
+      files: ""
+      exclude: ^$
+      types: []
+      types_or: []
+      exclude_types: []
+      always_run: false
+      fail_fast: false
+      verbose: false
+      pass_filenames: false
+      require_serial: false
+      description: "Sets the specified version of specified Python package."
+      language_version: default
+      minimum_pre_commit_version: "2.16.0"
+      args:
+      - "--package-dir=ci_cd"
+      - "--version=0.0.0"
+      - "--test"

.github/utils/run_hooks.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Run pre-commit hooks on all files in the repository.
+
+File used to test running the hooks in the CI/CD pipeline independently of the shell.
+"""
+from __future__ import annotations
+
+# import platform
+import subprocess  # nosec
+import sys
+
+SUCCESSFUL_FAILURES_MAPPING = {
+    "docs-api-reference": "The following files have been changed/added/removed:",
+    "docs-landing-page": "The landing page has been updated.",
+    "update-pyproject": "Successfully updated the following dependencies:",
+    "set-version": "Bumped version for ci_cd to 0.0.0.",
+}
+
+
+def main(hook: str, options: list[str]) -> None:
+    """Run pre-commit hooks on all files in the repository."""
+    run_pre_commit = (
+        "pre-commit run -c .github/utils/.pre-commit-config_testing.yaml "
+        "--all-files --verbose"
+    )
+
+    result = subprocess.run(
+        f"{run_pre_commit} {' '.join(_ for _ in options)} {hook}",
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        shell=True,  # nosec
+    )
+
+    if result.returncode != 0:
+        if SUCCESSFUL_FAILURES_MAPPING[hook] in result.stdout.decode():
+            print(f"Successfully failed {hook} hook.\n\n", flush=True)
+            print(result.stdout.decode(), flush=True)
+        else:
+            sys.exit(result.stdout.decode())
+    print(f"Successfully ran {hook} hook.\n\n", flush=True)
+    print(result.stdout.decode(), flush=True)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        raise sys.exit("Missing arguments")
+
+    # "Parse" arguments
+    # The first argument should be the hook name
+    if sys.argv[1] not in SUCCESSFUL_FAILURES_MAPPING:
+        raise sys.exit(
+            f"Invalid hook name: {sys.argv[1]}\n"
+            "The hook name should be the first argument. Any number of hook options "
+            "can then follow."
+        )
+
+    try:
+        main(
+            hook=sys.argv[1],
+            options=sys.argv[2:] if len(sys.argv) > 2 else [],
+        )
+    except Exception as exc:  # pylint: disable=broad-except
+        sys.exit(str(exc))

.github/workflows/_local_ci_tests.yml

@@ -35,12 +35,13 @@ jobs:
 
   pytest:
     name: pytest
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
+        os: ["ubuntu-latest", "windows-latest"]
 
     steps:
     - name: Checkout repository
@@ -66,3 +67,66 @@ jobs:
       with:
         fail_ci_if_error: true
         token: ${{ secrets.CODECOV_TOKEN }}
+
+  # These jobs are mainly to test a default run of the hooks including `--pre-commit`
+  run_hooks:
+    name: Run custom pre-commit hooks
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.10"]
+        os: ["ubuntu-latest", "windows-latest"]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version}}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version}}
+
+    - name: Install Python dependencies
+      run: |
+        python -m pip install -U pip
+        pip install -U setuptools wheel flit
+        pip install -e .
+        pip install -U pre-commit
+
+    # docs-api-reference
+    - name: Run docs-api-reference ('ci-cd create-api-reference-docs')
+      run: python .github/utils/run_hooks.py docs-api-reference
+
+    - name: Run docs-api-reference ('ci-cd create-api-reference-docs') (cmd)
+      if: runner.os == 'Windows'
+      run: python .github/utils/run_hooks.py docs-api-reference
+      shell: cmd
+
+    # docs-landing-page
+    - name: Run docs-landing-page ('ci-cd create-docs-index')
+      run: python .github/utils/run_hooks.py docs-landing-page
+
+    - name: Run docs-landing-page ('ci-cd create-docs-index') (cmd)
+      if: runner.os == 'Windows'
+      run: python .github/utils/run_hooks.py docs-landing-page
+      shell: cmd
+
+    # update-pyproject
+    - name: Run update-pyproject ('ci-cd update-deps')
+      run: python .github/utils/run_hooks.py update-pyproject
+
+    - name: Run update-pyproject ('ci-cd update-deps') (cmd)
+      if: runner.os == 'Windows'
+      run: python .github/utils/run_hooks.py update-pyproject
+      shell: cmd
+
+    # set-version
+    - name: Run 'ci-cd setver'
+      run: python .github/utils/run_hooks.py set-version
+
+    - name: Run 'ci-cd setver' (cmd)
+      if: runner.os == 'Windows'
+      run: python .github/utils/run_hooks.py set-version
+      shell: cmd

ci_cd/tasks/api_reference_docs.py

@@ -10,7 +10,7 @@
 import shutil
 import sys
 from collections import defaultdict
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING
 
 from invoke import task
@@ -131,6 +131,12 @@ def create_api_reference_docs(  # pylint: disable=too-many-locals,too-many-branc
     if not special_option:
         special_option: list[str] = []  # type: ignore[no-redef]
 
+    # Initialize user-given paths as pure POSIX paths
+    package_dir: list[PurePosixPath] = [PurePosixPath(_) for _ in package_dir]
+    root_repo_path: PurePosixPath = PurePosixPath(root_repo_path)  # type: ignore[no-redef]
+    docs_folder: PurePosixPath = PurePosixPath(docs_folder)  # type: ignore[no-redef]
+    full_docs_folder: list[PurePosixPath] = [PurePosixPath(_) for _ in full_docs_folder]  # type: ignore[no-redef]
+
     def write_file(full_path: Path, content: str) -> None:
         """Write file with `content` to `full_path`"""
         if full_path.exists():
@@ -141,14 +147,22 @@ def write_file(full_path: Path, content: str) -> None:
             del cached_content
         full_path.write_text(content, encoding="utf8")
 
+    if pre_commit:
+        # Ensure git is installed
+        result: "Result" = context.run("git --version", hide=True)
+        if result.exited != 0:
+            sys.exit(
+                "Git is not installed. Please install it before running this task."
+            )
+
     if pre_commit and root_repo_path == ".":
         # Use git to determine repo root
-        result: "Result" = context.run("git rev-parse --show-toplevel", hide=True)
-        root_repo_path = result.stdout.strip("\n")
+        result = context.run("git rev-parse --show-toplevel", hide=True)
+        root_repo_path = result.stdout.strip("\n")  # type: ignore[no-redef]
 
     root_repo_path: Path = Path(root_repo_path).resolve()  # type: ignore[no-redef]
-    package_dirs: list[Path] = [root_repo_path / _ for _ in package_dir]
-    docs_api_ref_dir = root_repo_path / docs_folder / "api_reference"
+    package_dirs: list[Path] = [Path(root_repo_path / _) for _ in package_dir]
+    docs_api_ref_dir = Path(root_repo_path / docs_folder / "api_reference")
 
     LOGGER.debug(
         """package_dirs: %s
@@ -197,11 +211,8 @@ def write_file(full_path: Path, content: str) -> None:
     if debug:
         print("special_options_files:", special_options_files, flush=True)
 
-    if any("/" in _ for _ in unwanted_folder + unwanted_file):
-        sys.exit(
-            "Unwanted folders and files may NOT be paths. A forward slash (/) was "
-            "found in some of them."
-        )
+    if any(os.sep in _ or "/" in _ for _ in unwanted_folder + unwanted_file):
+        sys.exit("Unwanted folders and files may NOT be paths.")
 
     pages_template = 'title: "{name}"\n'
     md_template = "# {name}\n\n::: {py_path}\n"
@@ -295,21 +306,24 @@ def write_file(full_path: Path, content: str) -> None:
                     package.relative_to(root_repo_path) if relative else package.name
                 )
                 py_path = (
-                    f"{py_path_root}/{filename.stem}".replace("/", ".")
+                    f"{py_path_root}/{filename.stem}"
                     if str(relpath) == "."
                     or (str(relpath) == package.name and not single_package)
-                    else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}".replace(
-                        "/", "."
-                    )
+                    else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}"
                 )
+
+                # Replace OS specific path separators with forward slashes before
+                # replacing that with dots (for Python import paths).
+                py_path = py_path.replace(os.sep, "/").replace("/", ".")
+
                 LOGGER.debug("filename: %s\npy_path: %s", filename, py_path)
                 if debug:
                     print("filename:", filename, flush=True)
                     print("py_path:", py_path, flush=True)
 
-                relative_file_path = (
+                relative_file_path = Path(
                     str(filename) if str(relpath) == "." else str(relpath / filename)
-                )
+                ).as_posix()
 
                 # For special files we want to include EVERYTHING, even if it doesn't
                 # have a doc-string
@@ -352,22 +366,22 @@ def write_file(full_path: Path, content: str) -> None:
         # Check if there have been any changes.
         # List changes if yes.
 
-        # NOTE: grep returns an exit code of 1 if it doesn't find anything
-        # (which will be good in this case).
-        # Concerning the weird last grep command see:
+        # NOTE: Concerning the weird regular expression, see:
         # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html
-        result: "Result" = context.run(  # type: ignore[no-redef]
+        result = context.run(
             f'git -C "{root_repo_path}" status --porcelain '
-            f"{docs_api_ref_dir.relative_to(root_repo_path)} | "
-            "grep -E '^[? MARC][?MD]' || exit 0",
+            f"{docs_api_ref_dir.relative_to(root_repo_path)}",
             hide=True,
         )
         if result.stdout:
-            sys.exit(
-                f"{Emoji.CURLY_LOOP.value} The following files have been "
-                f"changed/added/removed:\n\n{result.stdout}\nPlease stage them:\n\n"
-                f"  git add {docs_api_ref_dir.relative_to(root_repo_path)}"
-            )
+            for line in result.stdout.splitlines():
+                if re.match(r"^[? MARC][?MD]", line):
+                    sys.exit(
+                        f"{Emoji.CURLY_LOOP.value} The following files have been "
+                        f"changed/added/removed:\n\n{result.stdout}\n"
+                        "Please stage them:\n\n"
+                        f"  git add {docs_api_ref_dir.relative_to(root_repo_path)}"
+                    )
         print(
             f"{Emoji.CHECK_MARK.value} No changes - your API reference documentation "
             "is up-to-date !"

ci_cd/tasks/docs_index.py

@@ -2,6 +2,7 @@
 
 Create the documentation index (home) page from `README.md`.
 """
+import re
 import sys
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -87,22 +88,21 @@ def create_docs_index(  # pylint: disable=too-many-locals
         # Check if there have been any changes.
         # List changes if yes.
 
-        # NOTE: grep returns an exit code of 1 if it doesn't find anything
-        # (which will be good in this case).
-        # Concerning the weird last grep command see:
+        # NOTE: Concerning the weird regular expression, see:
         # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html
         result: "Result" = context.run(  # type: ignore[no-redef]
             f'git -C "{root_repo_path}" status --porcelain '
-            f"{docs_index.relative_to(root_repo_path)} | "
-            "grep -E '^[? MARC][?MD]' || exit 0",
+            f"{docs_index.relative_to(root_repo_path)}",
             hide=True,
         )
         if result.stdout:
-            sys.exit(
-                f"{Emoji.CURLY_LOOP.value} The landing page has been updated.\n\n"
-                "Please stage it:\n\n"
-                f"  git add {docs_index.relative_to(root_repo_path)}"
-            )
+            for line in result.stdout.splitlines():
+                if re.match(r"^[? MARC][?MD]", line):
+                    sys.exit(
+                        f"{Emoji.CURLY_LOOP.value} The landing page has been updated."
+                        "\n\nPlease stage it:\n\n"
+                        f"  git add {docs_index.relative_to(root_repo_path)}"
+                    )
         print(
             f"{Emoji.CHECK_MARK.value} No changes - your landing page is up-to-date !"
         )

ci_cd/utils.py

@@ -2,6 +2,7 @@
 More information on `invoke` can be found at [pyinvoke.org](http://www.pyinvoke.org/).
 """
 import logging
+import platform
 import re
 from enum import Enum
 from pathlib import Path
@@ -18,6 +19,16 @@
 class Emoji(str, Enum):
     """Unicode strings for certain emojis."""
 
+    def __new__(cls, value: str) -> "Emoji":
+        obj = str.__new__(cls, value)
+        if platform.system() == "Windows":
+            # Windows does not support unicode emojis, so we replace them with
+            # their corresponding unicode escape sequences
+            obj._value_ = value.encode("unicode_escape").decode("utf-8")
+        else:
+            obj._value_ = value
+        return obj
+
     PARTY_POPPER = "\U0001f389"
     CHECK_MARK = "\u2714"
     CROSS_MARK = "\u274c"

tests/tasks/test_api_reference_docs.py

@@ -479,7 +479,7 @@ def test_larger_package(tmp_path: "Path") -> None:
     ]:
         py_path = f"{package_dir.name}." + str(
             module_dir.relative_to(api_reference_folder)
-        ).replace("/", ".")
+        ).replace(os.sep, "/").replace("/", ".")
         assert (module_dir / ".pages").read_text(
             encoding="utf8"
         ) == f'title: "{module_dir.name}"\n', (
@@ -637,7 +637,7 @@ def test_larger_multi_packages(tmp_path: "Path") -> None:
         for module_dir in [package_dir / _ for _ in new_submodules]:
             py_path = f"{package_dir.name}." + str(
                 module_dir.relative_to(package_dir)
-            ).replace("/", ".")
+            ).replace(os.sep, "/").replace("/", ".")
             assert (module_dir / ".pages").read_text(
                 encoding="utf8"
             ) == f'title: "{module_dir.name}"\n', (