Add typesafety checks (#212)

* tests: add typesafety tests

* nox: add typesafety command

* upath: typing fixes for generated flavours

* upath: update _flavour_sources.py

* upath: more typing fixes

* typesafety: add more tests

* upath: typing fixes

* ci: add typesafety checks

* nox -s lint

* upath: exclude fsspec==2024.3.1 for now

* _strip_protocol signature changes break local fs tests
* Windows URI parsing has issues

* nox -s lint fixes

* typesafety: add pathlib interface tests

* upath: various typing fixes

* typesafety: fix mypy output issue on python3.8

* typesafety: check .link_to and .walk

.github/workflows/tests.yml

@@ -49,6 +49,24 @@ jobs:
     - name: Run tests
       run: nox -s tests-${{ matrix.fsspec || matrix.pyv }} -- --cov-report=xml
 
+  typesafety:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.pyv }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install nox
+        run: python -m pip install --upgrade nox
+
+      - name: Run typesafety checks
+        run: nox -s typesafety
+
   lint:
     runs-on: ubuntu-latest
 

dev/generate_flavours.py

@@ -43,6 +43,7 @@
 import logging
 import re
 from typing import Any
+from typing import Literal
 from typing import cast
 from urllib.parse import parse_qs
 from urllib.parse import urlsplit
@@ -67,6 +68,22 @@
 class FileSystemFlavourBase:
     """base class for the fsspec flavours"""
 
+    protocol: str | tuple[str, ...]
+    root_marker: Literal["/", ""]
+    sep: Literal["/"]
+
+    @classmethod
+    def _strip_protocol(cls, path):
+        raise NotImplementedError
+
+    @staticmethod
+    def _get_kwargs_from_urls(path):
+        raise NotImplementedError
+
+    @classmethod
+    def _parent(cls, path):
+        raise NotImplementedError
+
     def __init_subclass__(cls: Any, **kwargs):
         if isinstance(cls.protocol, str):
             protocols = (cls.protocol,)
@@ -99,12 +116,27 @@ def __init_subclass__(cls: Any, **kwargs):
 }
 
 
+def _fix_abstract_file_system(x: str) -> str:
+    x = re.sub(
+        "protocol = 'abstract'", "protocol: str | tuple[str, ...] = 'abstract'", x
+    )
+    x = re.sub("root_marker = ''", "root_marker: Literal['', '/'] = ''", x)
+    x = re.sub("sep = '/'", "sep: Literal['/'] = '/'", x)
+    return x
+
+
 def _fix_azure_blob_file_system(x: str) -> str:
-    return re.sub(
-        r"host = ops.get\(\"host\", None\)",
-        'host: str | None = ops.get("host", None)',
+    x = re.sub(
+        r"if isinstance\(path, list\):",
+        "if isinstance(path, list):  # type: ignore[unreachable]",
         x,
     )
+    x = re.sub(
+        r"(return \[.*\])",
+        r"\1  # type: ignore[unreachable]",
+        x,
+    )
+    return x
 
 
 def _fix_memfs_file_system(x: str) -> str:
@@ -115,6 +147,15 @@ def _fix_memfs_file_system(x: str) -> str:
     )
 
 
+def _fix_oss_file_system(x: str) -> str:
+    x = re.sub(
+        r"path_string: str = stringify_path\(path\)",
+        "path_string = stringify_path(path)",
+        x,
+    )
+    return x
+
+
 def _fix_xrootd_file_system(x: str) -> str:
     x = re.sub(
         r"client.URL",
@@ -129,8 +170,10 @@ def _fix_xrootd_file_system(x: str) -> str:
 
 
 FIX_SOURCE = {
+    "AbstractFileSystem": _fix_abstract_file_system,
     "AzureBlobFileSystem": _fix_azure_blob_file_system,
     "MemFS": _fix_memfs_file_system,
+    "OSSFileSystem": _fix_oss_file_system,
     "XRootDFileSystem": _fix_xrootd_file_system,
 }
 
@@ -303,7 +346,7 @@ def create_source() -> str:
             AbstractFileSystem,
             ["_strip_protocol", "_get_kwargs_from_urls", "_parent"],
             {},
-            ["protocol", "root_marker"],
+            ["protocol", "root_marker", "sep"],
             cls_suffix=BASE_CLASS_NAME_SUFFIX,
             base_cls="FileSystemFlavourBase",
         )

dev/requirements.txt

@@ -0,0 +1,18 @@
+fsspec[git,hdfs,dask,http,sftp,smb]==2024.2.0
+
+# these dependencies define their own filesystems
+adlfs==2024.2.0
+boxfs==0.2.1
+dropboxdrivefs==1.3.1
+gcsfs==2024.2.0
+s3fs==2024.2.0
+ocifs==1.3.1
+webdav4[fsspec]==0.9.8
+# gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ...
+morefs[asynclocalfs]==0.2.0
+dvc==3.47.0
+huggingface_hub==0.20.3
+lakefs-spec==0.7.0
+ossfs==2023.12.0
+fsspec-xrootd==0.2.4
+wandbfs==0.0.2

noxfile.py

@@ -91,6 +91,22 @@ def type_checking(session):
     session.run("python", "-m", "mypy")
 
 
+@nox.session
+def typesafety(session):
+    session.install("-e", ".[tests]")
+    session.run(
+        "python",
+        "-m",
+        "pytest",
+        "-v",
+        "-p",
+        "pytest-mypy-plugins",
+        "--mypy-pyproject-toml-file",
+        "pyproject.toml",
+        "typesafety",
+    )
+
+
 @nox.session()
 def smoke(session):
     print("please tun `nox -s tests` instead")

pyproject.toml

@@ -37,7 +37,7 @@ force_single_line = true
 line_length = 88
 
 [tool.pytest.ini_options]
-addopts = "-ra -m 'not hdfs'"
+addopts = "-ra -m 'not hdfs' -p no:pytest-mypy-plugins"
 markers = [
   "hdfs: mark test as hdfs",
   "pathlib: mark cpython pathlib tests",
@@ -61,7 +61,7 @@ exclude_lines = [
 
 [tool.mypy]
 # Error output
-show_column_numbers = true
+show_column_numbers = false
 show_error_codes = true
 show_error_context = true
 show_traceback = true

setup.cfg

@@ -24,7 +24,7 @@ python_requires = >=3.8
 zip_safe = False
 packages = find:
 install_requires=
-    fsspec>=2022.1.0
+    fsspec >=2022.1.0,!=2024.3.1
 
 [options.extras_require]
 tests =
@@ -33,7 +33,8 @@ tests =
     pytest-cov==4.1.0
     pytest-mock==3.12.0
     pylint==2.17.4
-    mypy==1.8.0
+    mypy==1.10.0
+    pytest-mypy-plugins==3.1.2
     packaging
 dev =
     %(tests)s