MAINT: Feature/zipline3dev (#18) (#202)

* Feature/zipline3dev (#18)

* pandas > 2.0 + * sqlalchemy > 2 migration
* numpy fixes

* fix pytest config

* pandas < 2 is no longer tested or supported

* fix ci yml pandas2

.devcontainer/requirements.txt

@@ -1,86 +1,100 @@
-alembic==1.8.1
+alembic==1.11.1
 appdirs==1.4.4
-attrs==22.1.0
 bcolz-zipline==1.2.6
-black==22.10.0
-Bottleneck==1.3.5
-certifi==2022.9.24
+beautifulsoup4==4.12.2
+black==22.12.0
+blosc2==2.2.4
+Bottleneck==1.3.7
+cachetools==5.3.1
+certifi==2023.5.7
+cffi==1.15.1
 cfgv==3.3.1
-charset-normalizer==2.1.1
-click==8.0.4
-contourpy==1.0.6
-coverage==6.5.0
+chardet==5.1.0
+charset-normalizer==3.1.0
+click==8.1.3
+colorama==0.4.6
+contourpy==1.0.7
+coverage==7.2.7
+cryptography==41.0.1
 cycler==0.11.0
+Cython==0.29.35
 distlib==0.3.6
-empyrical-reloaded==0.5.8
-exchange-calendars==3.3
+empyrical-reloaded==0.5.9
+exchange-calendars==4.2.8
 execnet==1.9.0
-filelock==3.8.0
-flake8==5.0.4
-fonttools==4.38.0
-greenlet==2.0.1
-h5py==3.7.0
-identify==2.5.9
+filelock==3.12.2
+flake8==6.0.0
+fonttools==4.39.4
+frozendict==2.3.8
+greenlet==2.0.2
+h5py==3.8.0
+html5lib==1.1
+identify==2.5.24
 idna==3.4
-iniconfig==1.1.1
+iniconfig==2.0.0
 intervaltree==3.1.0
 iso3166==2.1.1
 iso4217==1.11.20220401
 kiwisolver==1.4.4
 korean-lunar-calendar==0.3.1
-lru-dict==1.1.8
-lxml==4.9.1
+lru-dict==1.2.0
+lxml==4.9.2
 Mako==1.2.4
-MarkupSafe==2.1.1
-matplotlib==3.6.2
+MarkupSafe==2.1.3
+matplotlib==3.7.1
 mccabe==0.7.0
+msgpack==1.0.5
 multipledispatch==0.6.0
 multitasking==0.0.11
-mypy-extensions==0.4.3
-networkx==2.8.8
-nodeenv==1.7.0
+mypy-extensions==1.0.0
+ndindex==1.7
+networkx==3.1
+nodeenv==1.8.0
 numexpr==2.8.4
-numpy==1.23.5
-packaging==21.3
-pandas==1.5.1
+numpy==1.25.0
+packaging==23.1
+pandas==2.0.2
 pandas-datareader==0.10.0
-parameterized==0.8.1
-pathspec==0.10.2
+parameterized==0.9.0
+pathspec==0.11.1
 patsy==0.5.3
-Pillow==9.3.0
-platformdirs==2.5.4
+Pillow==9.5.0
+platformdirs==3.5.3
 pluggy==1.0.0
-pre-commit==2.20.0
-py==1.11.0
-pycodestyle==2.9.1
-pyflakes==2.5.0
-pyluach==2.0.2
+pre-commit==3.3.2
+py-cpuinfo==9.0.0
+pycodestyle==2.10.0
+pycparser==2.21
+pyflakes==3.0.1
+pyluach==2.2.0
 pyparsing==3.0.9
-pytest==6.2.5
-pytest-cov==4.0.0
-pytest-rerunfailures==10.3
+pyproject-api==1.5.2
+pytest==7.4.0
+pytest-cov==4.1.0
+pytest-rerunfailures==11.1.2
 pytest-timeout==2.1.0
-pytest-xdist==3.0.2
+pytest-xdist==3.3.1
 python-dateutil==2.8.2
 python-interface==1.6.1
-pytz==2022.6
+pytz==2023.3
 PyYAML==6.0
-requests==2.28.1
-responses==0.22.0
-scipy==1.9.3
+requests==2.31.0
+responses==0.23.1
+scipy==1.11.0
 six==1.16.0
 sortedcontainers==2.4.0
-SQLAlchemy==1.4.44
-statsmodels==0.13.5
-TA-Lib==0.4.25
-tables==3.7.0
-testfixtures==7.0.3
-toml==0.10.2
-tomli==2.0.1
+soupsieve==2.4.1
+SQLAlchemy==2.0.17
+statsmodels==0.14.0
+TA-Lib==0.4.26
+tables==3.8.0
+testfixtures==7.1.0
 toolz==0.12.0
-tox==3.27.1
-trading-calendars==2.1.1
-types-toml==0.10.8.1
-urllib3==1.26.12
-virtualenv==20.16.7
-yfinance==0.1.87
+tox==4.6.3
+types-PyYAML==6.0.12.10
+typing_extensions==4.6.3
+tzdata==2023.3
+urllib3==2.0.3
+virtualenv==20.23.1
+webencodings==0.5.1
+yfinance==0.2.22

.github/workflows/ci_tests_full.yml

@@ -35,7 +35,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout Zipline

.github/workflows/ci_tests_quick.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: flake8 Lint
         uses: py-actions/flake8@v2
@@ -37,7 +37,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
       - name: Checkout Zipline
@@ -80,7 +80,7 @@ jobs:
           max_attempts: 3
           retry_on: error
           new_command_on_retry: python -m pip install tox tox-gh-actions .[test]
-          command: tox -e py310-pandas15
+          command: tox -e py311-pandas2
 
       - name: Upload coverage data to Codecov
         if: ${{ matrix.os == 'ubuntu-latest' }}

pyproject.toml

@@ -46,16 +46,16 @@ dependencies = [
     'multipledispatch >=0.6.0',
     'networkx >=2.0',
     'numexpr >=2.6.1',
-    'numpy >=1.14.5, <1.24',
-    'pandas >=1.3, <1.6',
+    'numpy >=1.14.5',
+    'pandas >=2.0',
     'patsy >=0.4.0',
     'python-dateutil >=2.4.2',
     'python-interface >=1.5.3',
     'pytz >=2018.5',
     'requests >=2.9.1',
     'scipy >=0.17.1',
     'six >=1.10.0',
-    'sqlalchemy >=1.0.8A, <2',
+    'sqlalchemy >=2',
     'statsmodels >=0.6.1',
     'ta-lib >=0.4.09',
     'tables >=3.4.3',
@@ -134,9 +134,10 @@ local_scheme = 'dirty-tag'
 [tool.setuptools.package-data]
 "*" = ["*.pyi", "*.pyx", "*.pxi", "*.pxd"]
 
-[tool.pytest]
+[tool.pytest.ini_options]
 testpaths = 'tests'
 addopts = '-v'
+filterwarnings = 'ignore::DeprecationWarning:pandas_datareader.compat'
 
 [tool.cibuildwheel]
 test-extras = "test"
@@ -153,7 +154,7 @@ skip = "*musllinux*"
 
 [tool.black]
 line-length = 88
-target-version = ['py38', 'py39', 'py310']
+target-version = ['py38', 'py39', 'py310', 'py311']
 exclude = '''
 (
     asv_bench/env
@@ -170,7 +171,7 @@ exclude = '''
 [tool.tox]
 legacy_tox_ini = """
 [tox]
-envlist = py{38,39,310}-pandas{12,13,14,15}
+envlist = py{38,39,310,311}-pandas{2}
 isolated_build = True
 skip_missing_interpreters = True
 minversion = 3.23.0
@@ -180,6 +181,7 @@ python =
     3.8: py38
     3.9: py39
     3.10: py310
+    3.11: py311
 
 [testenv]
 usedevelop = True
@@ -189,10 +191,7 @@ setenv =
 changedir = tmp
 extras = test
 deps =
-    pandas12: pandas>=1.2.0,<1.3
-    pandas13: pandas>=1.3.0,<1.4
-    pandas14: pandas>=1.4.0,<1.5
-    pandas15: pandas>=1.5.0,<1.6
+    pandas2: pandas>=2.0
 
 commands =
     pytest -n 4 --reruns 5 --cov={toxinidir}/src --cov-report term  --cov-report=xml --cov-report=html:htmlcov {toxinidir}/tests

src/zipline/algorithm.py

@@ -16,7 +16,7 @@
 from collections import namedtuple
 from copy import copy
 import warnings
-from datetime import tzinfo, time
+from datetime import tzinfo, time, timezone
 import logging
 import pytz
 import pandas as pd
@@ -537,7 +537,6 @@ def _create_benchmark_source(self):
             benchmark_asset = self.asset_finder.retrieve_asset(self.benchmark_sid)
             benchmark_returns = None
         else:
-
             benchmark_asset = None
             benchmark_returns = self.benchmark_returns
         return BenchmarkSource(
@@ -644,7 +643,6 @@ def _create_daily_stats(self, perfs):
         # warning.
         for perf in perfs:
             if "daily_perf" in perf:
-
                 perf["daily_perf"].update(perf["daily_perf"].pop("recorded_vars"))
                 perf["daily_perf"].update(perf["cumulative_risk_metrics"])
                 daily_perfs.append(perf["daily_perf"])
@@ -778,7 +776,7 @@ def fetch_csv(
         post_func=None,
         date_column="date",
         date_format=None,
-        timezone=pytz.utc.zone,
+        timezone=str(timezone.utc),
         symbol=None,
         mask=True,
         symbol_column=None,
@@ -1455,7 +1453,7 @@ def get_datetime(self, tz=None):
             The current simulation datetime converted to ``tz``.
         """
         dt = self.datetime
-        assert dt.tzinfo == pytz.utc, "Algorithm should have a utc datetime"
+        assert dt.tzinfo == timezone.utc, "Algorithm should have a utc datetime"
         if tz is not None:
             dt = dt.astimezone(tz)
         return dt

src/zipline/assets/asset_db_migrations.py

@@ -76,10 +76,13 @@ def downgrade(engine, desired_version):
 
     # Check the version of the db at the engine
     with engine.begin() as conn:
-        metadata = sa.MetaData(conn)
-        metadata.reflect()
-        version_info_table = metadata.tables["version_info"]
-        starting_version = sa.select((version_info_table.c.version,)).scalar()
+        metadata_obj = sa.MetaData()
+        metadata_obj.reflect(conn)
+        version_info_table = metadata_obj.tables["version_info"]
+        # starting_version = sa.select((version_info_table.c.version,)).scalar()
+        starting_version = conn.execute(
+            sa.select(version_info_table.c.version)
+        ).scalar()
 
         # Check for accidental upgrade
         if starting_version < desired_version:
@@ -180,7 +183,6 @@ def _downgrade_v1(op):
 
     # Execute batch op to allow column modification in SQLite
     with op.batch_alter_table("futures_contracts") as batch_op:
-
         # Rename 'multiplier'
         batch_op.alter_column(
             column_name="multiplier", new_column_name="contract_multiplier"

src/zipline/assets/asset_writer.py

@@ -412,7 +412,7 @@ def check_version_info(conn, version_table, expected_version: int):
 
     Parameters
     ----------
-    conn : sa.Connection
+    conn : Connection
         The connection to use to perform the check.
     version_table : sa.Table
         The version table of the asset database
@@ -424,7 +424,6 @@ def check_version_info(conn, version_table, expected_version: int):
     AssetDBVersionError
         If the version is in the table and not equal to ASSET_DB_VERSION.
     """
-
     # Read the version out of the table
     version_from_table = conn.execute(sa.select(version_table.c.version)).scalar()