diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 1a914473c1..4c758d7601 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -84,16 +84,6 @@ jobs: env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} - services: - redis: - image: redis - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@v2 with: diff --git a/.gitignore b/.gitignore index e4becb3c31..97a9d2c975 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ vendor .terraform/ *.tfvars -# python +# python # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -107,6 +107,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +infra/scripts/*.conf # Translations *.mo diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index d32740adf3..6225f2d1d1 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -41,7 +41,7 @@ try: from redis import Redis - from redis.cluster import ClusterNode, RedisCluster + from rediscluster import RedisCluster except ImportError as e: from feast.errors import FeastExtrasDependencyImportError @@ -163,9 +163,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): online_store_config.connection_string ) if online_store_config.redis_type == RedisType.redis_cluster: - kwargs["startup_nodes"] = [ - ClusterNode(**node) for node in startup_nodes - ] + kwargs["startup_nodes"] = startup_nodes self._client = RedisCluster(**kwargs) else: kwargs["host"] = startup_nodes[0]["host"] diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index d5f654e515..050600916a 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -54,14 +54,14 @@ attrs==21.4.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.21.1 +azure-core==1.23.0 # via # adlfs # azure-identity # azure-storage-blob azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.7.1 +azure-identity==1.8.0 # via adlfs azure-storage-blob==12.9.0 # via adlfs @@ -69,7 +69,7 @@ babel==2.9.1 # via sphinx backcall==0.2.0 # via ipython -backports.zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal @@ -77,11 +77,11 @@ black==19.10b0 # via feast (setup.py) bleach==4.1.0 # via nbconvert -boto3==1.20.46 +boto3==1.21.12 # via # feast (setup.py) # moto -botocore==1.23.46 +botocore==1.24.12 # via # boto3 # moto @@ -104,12 +104,12 @@ cffi==1.15.0 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via # aiohttp # requests # snowflake-connector-python -click==8.0.3 +click==8.0.4 # via # black # feast (setup.py) @@ -118,7 +118,7 @@ click==8.0.3 # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.3 +coverage[toml]==6.3.2 # via pytest-cov cryptography==3.3.2 # via @@ -139,8 +139,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -155,20 +153,20 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -entrypoints==0.3 +entrypoints==0.4 # via # altair # jupyter-client # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.73.0 +fastapi==0.74.1 # via feast (setup.py) -fastavro==1.4.9 +fastavro==1.4.10 # via # feast (setup.py) # pandavro -filelock==3.4.2 +filelock==3.6.0 # via virtualenv firebase-admin==4.5.2 # via feast (setup.py) @@ -178,11 +176,11 @@ frozenlist==1.3.0 # via # aiohttp # aiosignal -fsspec==2022.1.0 +fsspec==2022.2.0 # via # adlfs # gcsfs -gcsfs==2022.1.0 +gcsfs==2022.2.0 # via feast (setup.py) google-api-core[grpc]==1.31.5 # via @@ -194,7 +192,7 @@ google-api-core[grpc]==1.31.5 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.36.0 +google-api-python-client==2.39.0 # via firebase-admin google-auth==1.35.0 # via @@ -207,11 +205,11 @@ google-auth==1.35.0 # google-cloud-storage google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.4.6 +google-auth-oauthlib==0.5.0 # via gcsfs -google-cloud-bigquery==2.32.0 +google-cloud-bigquery==2.34.1 # via feast (setup.py) -google-cloud-bigquery-storage==2.11.0 +google-cloud-bigquery-storage==2.12.0 # via feast (setup.py) google-cloud-core==1.7.2 # via @@ -220,7 +218,7 @@ google-cloud-core==1.7.2 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.4.0 +google-cloud-datastore==2.5.0 # via feast (setup.py) google-cloud-firestore==2.3.4 # via firebase-admin @@ -240,9 +238,9 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata -great-expectations==0.14.4 +great-expectations==0.14.9 # via feast (setup.py) -grpcio==1.43.0 +grpcio==1.44.0 # via # feast (setup.py) # google-api-core @@ -250,7 +248,7 @@ grpcio==1.43.0 # grpcio-reflection # grpcio-testing # grpcio-tools -grpcio-reflection==1.43.0 +grpcio-reflection==1.44.0 # via feast (setup.py) grpcio-testing==1.34.0 # via feast (setup.py) @@ -260,13 +258,13 @@ h11==0.13.0 # via uvicorn hiredis==2.0.0 # via feast (setup.py) -httplib2==0.20.2 +httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.7 +identify==2.4.11 # via pre-commit idna==3.3 # via @@ -287,17 +285,16 @@ importlib-metadata==4.2.0 # pluggy # pre-commit # pytest - # redis # virtualenv importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 # via pytest -ipykernel==6.7.0 +ipykernel==6.9.1 # via # ipywidgets # notebook -ipython==7.31.1 +ipython==7.32.0 # via # ipykernel # ipywidgets @@ -342,7 +339,7 @@ jupyter-client==7.1.2 # ipykernel # nbclient # notebook -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -352,11 +349,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert jupyterlab-widgets==1.0.2 # via ipywidgets -libcst==0.4.1 - # via - # google-cloud-bigquery-storage - # google-cloud-datastore -markupsafe==2.0.1 +markupsafe==2.1.0 # via # jinja2 # moto @@ -376,9 +369,9 @@ mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.2 +moto==3.0.5 # via feast (setup.py) -msal==1.16.0 +msal==1.17.0 # via # azure-identity # msal-extensions @@ -399,14 +392,12 @@ multidict==6.0.2 mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 - # via - # mypy - # typing-inspect + # via mypy mypy-protobuf==3.1.0 # via feast (setup.py) -nbclient==0.5.10 +nbclient==0.5.11 # via nbconvert -nbconvert==6.4.1 +nbconvert==6.4.2 # via notebook nbformat==5.1.3 # via @@ -444,7 +435,6 @@ packaging==21.3 # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -461,7 +451,7 @@ parso==0.8.3 # via jedi pathspec==0.9.0 # via black -pbr==5.8.0 +pbr==5.8.1 # via mock pep517==0.12.0 # via pip-tools @@ -469,19 +459,19 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.4.0 +pip-tools==6.5.1 # via feast (setup.py) -platformdirs==2.4.1 +platformdirs==2.5.1 # via virtualenv pluggy==1.0.0 # via pytest -portalocker==2.3.2 +portalocker==2.4.0 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) prometheus-client==0.13.1 # via notebook -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython proto-plus==1.19.6 # via @@ -526,7 +516,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.14.0 +pycryptodomex==3.14.1 # via snowflake-connector-python pydantic==1.9.0 # via @@ -554,7 +544,7 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -pytest==6.2.5 +pytest==7.0.1 # via # feast (setup.py) # pytest-benchmark @@ -605,16 +595,19 @@ pytz-deprecation-shim==0.1.0.post0 pyyaml==6.0 # via # feast (setup.py) - # libcst # pre-commit # uvicorn pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) -regex==2022.1.18 +regex==2022.3.2 # via black requests==2.27.1 # via @@ -640,15 +633,15 @@ requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest -responses==0.17.0 +responses==0.18.0 # via moto rsa==4.8 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml -s3transfer==0.5.0 +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml +s3transfer==0.5.2 # via boto3 scipy==1.7.3 # via great-expectations @@ -673,13 +666,12 @@ six==1.16.0 # pandavro # pyopenssl # python-dateutil - # responses # virtualenv sniffio==1.2.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.3 +snowflake-connector-python[pandas]==2.7.4 # via feast (setup.py) sphinx==4.3.2 # via @@ -705,27 +697,27 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.6.0 +tensorflow-metadata==1.7.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations -terminado==0.13.1 +terminado==0.13.2 # via notebook testcontainers==3.4.2 # via feast (setup.py) -testpath==0.5.0 +testpath==0.6.0 # via nbconvert toml==0.10.2 # via # black # feast (setup.py) # pre-commit - # pytest -tomli==2.0.0 +tomli==2.0.1 # via # coverage # mypy # pep517 + # pytest toolz==0.11.2 # via altair tornado==6.1 @@ -734,7 +726,7 @@ tornado==6.1 # jupyter-client # notebook # terminado -tqdm==4.62.3 +tqdm==4.63.0 # via # feast (setup.py) # great-expectations @@ -754,48 +746,43 @@ typed-ast==1.5.2 # via # black # mypy -types-futures==3.3.8 - # via types-protobuf -types-protobuf==3.19.7 +types-protobuf==3.19.12 # via # feast (setup.py) # mypy-protobuf types-python-dateutil==2.8.9 # via feast (setup.py) -types-pytz==2021.3.4 +types-pytz==2021.3.5 # via feast (setup.py) types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.13 +types-redis==4.1.17 # via feast (setup.py) -types-requests==2.27.8 +types-requests==2.27.11 # via feast (setup.py) -types-setuptools==57.4.8 +types-setuptools==57.4.9 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.8 +types-urllib3==1.26.10 # via types-requests -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # aiohttp # anyio # argon2-cffi # asgiref # async-timeout + # azure-core # great-expectations # h11 # importlib-metadata # jsonschema - # libcst # mypy # pydantic # starlette - # typing-inspect # uvicorn # yarl -typing-inspect==0.7.1 - # via libcst tzdata==2021.5 # via pytz-deprecation-shim tzlocal==4.1 @@ -809,11 +796,11 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.1 +uvicorn[standard]==0.17.5 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.13.0 +virtualenv==20.13.2 # via pre-commit watchgod==0.7 # via uvicorn @@ -821,20 +808,18 @@ wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 # via bleach -websocket-client==1.2.3 +websocket-client==1.3.1 # via docker -websockets==10.1 +websockets==10.2 # via uvicorn -werkzeug==2.0.2 +werkzeug==2.0.3 # via moto wheel==0.37.1 # via pip-tools widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 7a94294c95..5a8aa35532 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -116,6 +114,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +131,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.1.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +181,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -356,6 +346,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +431,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +451,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -727,7 +726,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +753,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +778,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,9 +821,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -843,7 +830,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 1421d7e3c3..403aab70aa 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -69,10 +67,6 @@ babel==2.9.1 # via sphinx backcall==0.2.0 # via ipython -backports.zoneinfo==0.2.1 - # via - # pytz-deprecation-shim - # tzlocal black==19.10b0 # via feast (setup.py) bleach==4.1.0 @@ -116,6 +110,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +127,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.1.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +137,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +177,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,20 +274,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv -importlib-resources==5.4.0 - # via jsonschema + # via great-expectations iniconfig==1.1.1 # via pytest ipykernel==6.7.0 @@ -356,6 +340,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +425,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +445,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +592,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +601,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -644,10 +637,10 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3transfer==0.5.0 # via boto3 scipy==1.7.3 @@ -727,7 +720,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +747,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +772,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,18 +815,13 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 # via aiohttp zipp==3.7.0 - # via - # importlib-metadata - # importlib-resources - # pep517 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 7535987f83..8c58911663 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -77,7 +77,8 @@ ] REDIS_REQUIRED = [ - "redis>=4.1.0", + "redis==3.5.3", + "redis-py-cluster>=2.1.3", "hiredis>=2.0.0", ] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index a342884468..9de146ac12 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -30,6 +30,7 @@ ) from tests.integration.feature_repos.repo_configuration import ( FULL_REPO_CONFIGS, + REDIS_CLUSTER_CONFIG, REDIS_CONFIG, Environment, TestData, @@ -169,10 +170,14 @@ def cleanup(): return e -@pytest.fixture() +@pytest.fixture( + params=[REDIS_CONFIG, REDIS_CLUSTER_CONFIG], + scope="session", + ids=[str(c) for c in [REDIS_CONFIG, REDIS_CLUSTER_CONFIG]], +) def local_redis_environment(request, worker_id): e = construct_test_environment( - IntegrationTestRepoConfig(online_store=REDIS_CONFIG), worker_id=worker_id + IntegrationTestRepoConfig(online_store=request.param), worker_id=worker_id ) def cleanup(): @@ -192,6 +197,16 @@ def cleanup(): return construct_universal_test_data(environment) +@pytest.fixture(scope="session") +def redis_universal_data_sources(request, local_redis_environment): + def cleanup(): + # logger.info("Running cleanup in %s, Request: %s", worker_id, request.param) + local_redis_environment.data_source_creator.teardown() + + request.addfinalizer(cleanup) + return construct_universal_test_data(local_redis_environment) + + @pytest.fixture(scope="session") def e2e_data_sources(environment: Environment, request): df = create_dataset() diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index c9de56b76a..2a2ad93e44 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -47,7 +47,7 @@ DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} # Port 12345 will chosen as default for redis node configuration because Redis Cluster is started off of nodes 6379 -> 6384. This causes conflicts in cli integration tests so we manually keep them separate. -REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} +REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:12345,db=0"} REDIS_CLUSTER_CONFIG = { "type": "redis", "redis_type": "redis_cluster", @@ -72,6 +72,7 @@ [ # Redis configurations IntegrationTestRepoConfig(online_store=REDIS_CONFIG), + IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG), # GCP configurations IntegrationTestRepoConfig( provider="gcp", diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 0e2e9da9e9..dc41bf19de 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -34,13 +34,13 @@ @pytest.mark.integration -def test_entity_ttl_online_store(local_redis_environment, universal_data_sources): +def test_entity_ttl_online_store(local_redis_environment, redis_universal_data_sources): if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": return fs = local_redis_environment.feature_store # setting ttl setting in online store to 1 second fs.config.online_store.key_ttl_seconds = 1 - entities, datasets, data_sources = universal_data_sources + entities, datasets, data_sources = redis_universal_data_sources driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver) driver_entity = driver()