Python 3.11 support

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10.6']
+        python-version: ['3.8', '3.9', '3.10.6', '3.11']
 
     steps:
     - uses: actions/checkout@v3

pandas-stubs/core/arrays/arrow/dtype.pyi

@@ -1,4 +1,12 @@
-import pyarrow as pa
+import sys
+from typing import Any
+
+from typing_extensions import TypeAlias
+
+if sys.version_info < (3, 11):
+    import pyarrow as pa
+else:
+    pa: TypeAlias = Any
 
 from pandas.core.dtypes.base import StorageExtensionDtype
 

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Topic :: Scientific/Engineering"
 ]
 packages = [
@@ -30,14 +31,14 @@ packages = [
 "Documentation" = "https://pandas.pydata.org/pandas-docs/stable"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.11"
+python = ">=3.8,<3.12"
 types-pytz = ">= 2022.1.1"
 
 [tool.poetry.dev-dependencies]
 mypy = "0.990"
-pyarrow = ">=9.0.0"
+pyarrow = { version = ">=9.0.0", python = "<3.11" }
 pytest = ">=7.1.2"
-pyright = ">=1.1.278"
+pyright = "<=1.1.279"
 poethepoet = "0.16.0"
 loguru = ">=0.6.0"
 pandas = "1.5.1"
@@ -47,9 +48,9 @@ pre-commit = ">=2.19.0"
 black = ">=22.8.0"
 isort = ">=5.10.1"
 openpyxl = ">=3.0.10"
-tables = ">=3.7.0"
-lxml = ">=4.7.1,<4.9.0"
-pyreadstat = ">=1.1.9"
+tables = { version = ">=3.7.0", python = "<3.11" }
+lxml = { version = ">=4.7.1,<4.9.0", python = "<3.11" }
+pyreadstat = ">=1.2.0"
 xlrd = ">=2.0.1"
 pyxlsb = ">=1.0.9"
 odfpy = ">=1.4.1"

tests/__init__.py

@@ -6,6 +6,7 @@
 )
 import os
 import platform
+import sys
 from typing import (
     TYPE_CHECKING,
     Final,
@@ -21,6 +22,21 @@
 WINDOWS = os.name == "nt" or "cygwin" in platform.system().lower()
 PD_LTE_15 = Version(pd.__version__) < Version("1.5.999")
 
+arrow_skip = pytest.mark.skipif(
+    sys.version_info >= (3, 11), reason="pyarrow is not available for 3.11 yet"
+)
+# This is only needed temporarily due to no wheels being available for arrow on 3.11
+
+lxml_skip = pytest.mark.skipif(
+    sys.version_info >= (3, 11), reason="lxml is not available for 3.11 yet"
+)
+# This is only needed temporarily due to no wheels being available for lxml on 3.11
+
+pytables_skip = pytest.mark.skipif(
+    sys.version_info >= (3, 11), reason="pytables is not available for 3.11 yet"
+)
+# This is only needed temporarily due to no wheels being available for pytables on 3.11
+
 
 def check(actual: T, klass: type, dtype: type | None = None, attr: str = "left") -> T:
     if not isinstance(actual, klass):
@@ -44,6 +60,7 @@ def pytest_warns_bounded(
     match: str,
     lower: str | None = None,
     upper: str | None = None,
+    version_str: str | None = None,
 ) -> AbstractContextManager:
     """
     Version conditional pytest.warns context manager
@@ -62,11 +79,14 @@ def pytest_warns_bounded(
         The lower bound of the version to check for the warning.
     upper : str, optional
         The upper bound of the version to check for the warning.
+    version_str: str, optional
+        The version string to use.  If None, then uses the pandas version.
+        Can be used to check a python version as well
 
     Notes
     -----
     The lower and upper bounds are exclusive so that a pytest.warns context
-    manager is returned if lower < pd.__version__ < upper.
+    manager is returned if lower < version_str < upper.
 
     Examples
     --------
@@ -85,10 +105,21 @@ def pytest_warns_bounded(
     ):
         # Versions between 1.3.x and 1.5.x will raise an error
         pass
+
+    with pytest_warns_bounded(
+        UserWarning, match="foo", lower="3.10",
+        version_str = platform.python_version()
+    ):
+        # Python version 3.11 and above will raise an error
+        # if the warning is not issued
+        pass
     """
     lb = Version("0.0.0") if lower is None else Version(lower)
     ub = Version("9999.0.0") if upper is None else Version(upper)
-    current = Version(pd.__version__)
+    if version_str is None:
+        current = Version(pd.__version__)
+    else:
+        current = Version(version_str)
     if lb < current < ub:
         return pytest.warns(warning, match=match)
     else:

tests/test_frame.py

@@ -6,6 +6,7 @@
 import io
 import itertools
 from pathlib import Path
+import platform
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -1757,7 +1758,15 @@ class MyDataFrame(pd.DataFrame, Generic[T]):
     def func() -> MyDataFrame[int]:
         return MyDataFrame[int]({"foo": [1, 2, 3]})
 
-    func()
+    # This should be fixed in pandas 1.5.2, if
+    # https://github.com/pandas-dev/pandas/pull/49736 is included
+    with pytest_warns_bounded(
+        UserWarning,
+        "Pandas doesn't allow columns to be created",
+        lower="3.10.99",
+        version_str=platform.python_version(),
+    ):
+        func()
 
 
 def test_to_xarray():

tests/test_io.py

@@ -65,17 +65,25 @@
 from pandas.io.sas.sas_xport import XportReader
 from pandas.io.stata import StataReader
 
+from . import (
+    arrow_skip,
+    lxml_skip,
+    pytables_skip,
+)
+
 DF = DataFrame({"a": [1, 2, 3], "b": [0.0, 0.0, 0.0]})
 CWD = os.path.split(os.path.abspath(__file__))[0]
 
 
+@arrow_skip
 @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows")
 def test_orc():
     with ensure_clean() as path:
         check(assert_type(DF.to_orc(path), None), type(None))
         check(assert_type(read_orc(path), DataFrame), DataFrame)
 
 
+@arrow_skip
 @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows")
 def test_orc_path():
     with ensure_clean() as path:
@@ -84,6 +92,7 @@ def test_orc_path():
         check(assert_type(read_orc(pathlib_path), DataFrame), DataFrame)
 
 
+@arrow_skip
 @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows")
 def test_orc_buffer():
     with ensure_clean() as path:
@@ -96,18 +105,21 @@ def test_orc_buffer():
         file_r.close()
 
 
+@arrow_skip
 @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows")
 def test_orc_columns():
     with ensure_clean() as path:
         check(assert_type(DF.to_orc(path, index=False), None), type(None))
         check(assert_type(read_orc(path, columns=["a"]), DataFrame), DataFrame)
 
 
+@arrow_skip
 @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows")
 def test_orc_bytes():
     check(assert_type(DF.to_orc(index=False), bytes), bytes)
 
 
+@lxml_skip
 def test_xml():
     with ensure_clean() as path:
         check(assert_type(DF.to_xml(path), None), type(None))
@@ -116,6 +128,7 @@ def test_xml():
             check(assert_type(read_xml(f), DataFrame), DataFrame)
 
 
+@lxml_skip
 def test_xml_str():
     with ensure_clean() as path:
         check(assert_type(DF.to_xml(), str), str)
@@ -282,12 +295,14 @@ def test_sas_xport() -> None:
         pass
 
 
+@pytables_skip
 def test_hdf():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df"), None), type(None))
         check(assert_type(read_hdf(path), Union[DataFrame, Series]), DataFrame)
 
 
+@pytables_skip
 def test_hdfstore():
     with ensure_clean() as path:
         store = HDFStore(path, model="w")
@@ -329,6 +344,7 @@ def test_hdfstore():
         store.close()
 
 
+@pytables_skip
 def test_read_hdf_iterator():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None))
@@ -343,6 +359,7 @@ def test_read_hdf_iterator():
         ti.close()
 
 
+@pytables_skip
 def test_hdf_context_manager():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None))
@@ -351,6 +368,7 @@ def test_hdf_context_manager():
             check(assert_type(store.get("df"), Union[DataFrame, Series]), DataFrame)
 
 
+@pytables_skip
 def test_hdf_series():
     s = DF["a"]
     with ensure_clean() as path:
@@ -392,13 +410,15 @@ def test_json_chunk():
     check(assert_type(DF.to_json(), str), str)
 
 
+@arrow_skip
 def test_parquet():
     with ensure_clean() as path:
         check(assert_type(DF.to_parquet(path), None), type(None))
         check(assert_type(read_parquet(path), DataFrame), DataFrame)
     check(assert_type(DF.to_parquet(), bytes), bytes)
 
 
+@arrow_skip
 def test_parquet_options():
     with ensure_clean(".parquet") as path:
         check(
@@ -408,6 +428,7 @@ def test_parquet_options():
         check(assert_type(read_parquet(path), DataFrame), DataFrame)
 
 
+@arrow_skip
 def test_feather():
     with ensure_clean() as path:
         check(assert_type(DF.to_feather(path), None), type(None))
@@ -795,6 +816,7 @@ def test_read_sql_query_generator():
         con.close()
 
 
+@lxml_skip
 def test_read_html():
     check(assert_type(DF.to_html(), str), str)
     with ensure_clean() as path:

tests/test_pandas.py

@@ -2,6 +2,7 @@
 
 import datetime as dt
 import random
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -464,14 +465,15 @@ def test_crosstab() -> None:
 def test_arrow_dtype() -> None:
     pytest.importorskip("pyarrow")
 
-    import pyarrow as pa
+    if sys.version_info < (3, 11):
+        import pyarrow as pa
 
-    check(
-        assert_type(
-            pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype
-        ),
-        pd.ArrowDtype,
-    )
+        check(
+            assert_type(
+                pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype
+            ),
+            pd.ArrowDtype,
+        )
 
 
 def test_hashing():