diff --git a/sdk/python/feast/expediagroup/vectordb/milvus_online_store.py b/sdk/python/feast/expediagroup/vectordb/milvus_online_store.py new file mode 100644 index 0000000000..cb50e6c6a0 --- /dev/null +++ b/sdk/python/feast/expediagroup/vectordb/milvus_online_store.py @@ -0,0 +1,73 @@ +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from pydantic.typing import Literal + +from feast import Entity, RepoConfig +from feast.expediagroup.vectordb.vector_feature_view import VectorFeatureView +from feast.expediagroup.vectordb.vector_online_store import VectorOnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel + + +class MilvusOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for the Milvus online store""" + + type: Literal["milvus"] = "milvus" + """Online store type selector""" + + host: str + """ the host URL """ + + port: int = 19530 + """ the port to connect to a Milvus instance. Should be the one used for GRPC (default: 19530) """ + + +class MilvusOnlineStore(VectorOnlineStore): + def online_write_batch( + self, + config: RepoConfig, + table: VectorFeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + raise NotImplementedError( + "to be implemented in https://jira.expedia.biz/browse/EAPC-7971" + ) + + def online_read( + self, + config: RepoConfig, + table: VectorFeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + raise NotImplementedError( + "to be implemented in https://jira.expedia.biz/browse/EAPC-7972" + ) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[VectorFeatureView], + tables_to_keep: Sequence[VectorFeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + raise NotImplementedError( + "to be implemented in https://jira.expedia.biz/browse/EAPC-7970" + ) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[VectorFeatureView], + entities: Sequence[Entity], + ): + raise NotImplementedError( + "to be implemented in https://jira.expedia.biz/browse/EAPC-7974" + ) diff --git a/sdk/python/feast/expediagroup/vectordb/vector_online_store.py b/sdk/python/feast/expediagroup/vectordb/vector_online_store.py new file mode 100644 index 0000000000..e231a3c917 --- /dev/null +++ b/sdk/python/feast/expediagroup/vectordb/vector_online_store.py @@ -0,0 +1,63 @@ +from abc import abstractmethod +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from feast import Entity, RepoConfig +from feast.expediagroup.vectordb.vector_feature_view import VectorFeatureView +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto + + +class VectorOnlineStore(OnlineStore): + """ + Abstraction for vector database implementations of the online store interface. Any online store implementation of + a vector database should inherit this class. + """ + + @abstractmethod + def online_write_batch( + self, + config: RepoConfig, + table: VectorFeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + # to be implemented in inheriting class + pass + + @abstractmethod + def online_read( + self, + config: RepoConfig, + table: VectorFeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + # to be implemented in inheriting class + pass + + @abstractmethod + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[VectorFeatureView], + tables_to_keep: Sequence[VectorFeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + # to be implemented in inheriting class + pass + + @abstractmethod + def teardown( + self, + config: RepoConfig, + tables: Sequence[VectorFeatureView], + entities: Sequence[Entity], + ): + # to be implemented in inheriting class + pass diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 8827278d0c..a6320c55d9 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -64,6 +64,7 @@ "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore", "rockset": "feast.infra.online_stores.contrib.rockset_online_store.rockset.RocksetOnlineStore", "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore", + "milvus": "feast.expediagroup.vectordb.milvus_online_store.MilvusOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 3aea1837fb..8fe28cc882 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -7,8 +7,8 @@ adal==1.2.7 # via msrestazure adlfs==0.5.9 - # via feast (setup.py) -aiohttp==3.8.4 + # via eg-feast (setup.py) +aiohttp==3.8.5 # via # adlfs # gcsfs @@ -18,7 +18,7 @@ alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations -anyio==3.7.0 +anyio==3.7.1 # via # httpcore # jupyter-server @@ -31,10 +31,7 @@ appnope==0.1.3 # ipykernel # ipython argon2-cffi==21.3.0 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 @@ -44,9 +41,11 @@ asn1crypto==1.5.1 # oscrypto # snowflake-connector-python assertpy==1.1 - # via feast (setup.py) + # via eg-feast (setup.py) asttokens==2.2.1 # via stack-data +async-lru==2.0.4 + # via jupyterlab async-timeout==4.0.2 # via # aiohttp @@ -56,9 +55,10 @@ attrs==23.1.0 # aiohttp # bowler # jsonschema + # referencing avro==1.10.0 - # via feast (setup.py) -azure-core==1.27.0 + # via eg-feast (setup.py) +azure-core==1.28.0 # via # adlfs # azure-identity @@ -69,45 +69,47 @@ azure-datalake-store==0.0.53 azure-identity==1.13.0 # via # adlfs - # feast (setup.py) -azure-storage-blob==12.16.0 + # eg-feast (setup.py) +azure-storage-blob==12.17.0 # via # adlfs - # feast (setup.py) + # eg-feast (setup.py) babel==2.12.1 - # via sphinx + # via + # jupyterlab-server + # sphinx backcall==0.2.0 # via ipython beautifulsoup4==4.12.2 # via nbconvert black==22.12.0 - # via feast (setup.py) + # via eg-feast (setup.py) bleach==6.0.0 # via nbconvert -boto3==1.26.146 +boto3==1.28.16 # via - # feast (setup.py) + # eg-feast (setup.py) # moto -botocore==1.29.146 +botocore==1.31.16 # via # boto3 # moto # s3transfer bowler==0.9.0 - # via feast (setup.py) + # via eg-feast (setup.py) build==0.10.0 # via - # feast (setup.py) + # eg-feast (setup.py) # pip-tools bytewax==0.15.1 - # via feast (setup.py) -cachecontrol==0.13.0 + # via eg-feast (setup.py) +cachecontrol==0.13.1 # via firebase-admin cachetools==5.3.1 # via google-auth -cassandra-driver==3.27.0 - # via feast (setup.py) -certifi==2023.5.7 +cassandra-driver==3.28.0 + # via eg-feast (setup.py) +certifi==2023.7.22 # via # httpcore # httpx @@ -124,17 +126,17 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests # snowflake-connector-python -click==8.1.3 +click==8.1.6 # via # black # bowler # dask - # feast (setup.py) + # eg-feast (setup.py) # geomet # great-expectations # moreorless @@ -144,7 +146,7 @@ cloudpickle==2.2.1 # via dask colorama==0.4.6 # via - # feast (setup.py) + # eg-feast (setup.py) # great-expectations comm==0.1.3 # via ipykernel @@ -155,8 +157,7 @@ cryptography==40.0.2 # adal # azure-identity # azure-storage-blob - # cassandra-driver - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # moto # msal @@ -165,8 +166,8 @@ cryptography==40.0.2 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.5.1 - # via feast (setup.py) +dask==2023.7.1 + # via eg-feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery debugpy==1.6.7 @@ -181,50 +182,52 @@ deprecated==1.2.14 # via redis deprecation==2.1.0 # via testcontainers -dill==0.3.6 +dill==0.3.7 # via # bytewax - # feast (setup.py) + # eg-feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via - # feast (setup.py) + # eg-feast (setup.py) # testcontainers docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.1 +environs==9.5.0 + # via pymilvus +exceptiongroup==1.1.2 # via # anyio # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist executing==1.2.0 # via stack-data -fastapi==0.95.2 - # via feast (setup.py) -fastavro==1.7.4 +fastapi==0.100.1 + # via eg-feast (setup.py) +fastavro==1.8.2 # via - # feast (setup.py) + # eg-feast (setup.py) # pandavro -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 # via nbformat -filelock==3.12.0 +filelock==3.12.2 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) + # via eg-feast (setup.py) fissix==21.11.13 # via bowler -flake8==6.0.0 - # via feast (setup.py) +flake8==6.1.0 + # via eg-feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -234,14 +237,14 @@ fsspec==2022.1.0 # dask # gcsfs gcsfs==2022.1.0 - # via feast (setup.py) + # via eg-feast (setup.py) geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.0 +google-api-core[grpc]==2.11.1 # via - # feast (setup.py) + # eg-feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -251,9 +254,9 @@ google-api-core[grpc]==2.11.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.88.0 +google-api-python-client==2.95.0 # via firebase-admin -google-auth==2.19.1 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -267,26 +270,26 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-bigquery[pandas]==3.11.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.20.0 - # via feast (setup.py) -google-cloud-bigtable==2.18.1 - # via feast (setup.py) -google-cloud-core==2.3.2 +google-cloud-bigquery[pandas]==3.11.4 + # via eg-feast (setup.py) +google-cloud-bigquery-storage==2.22.0 + # via eg-feast (setup.py) +google-cloud-bigtable==2.20.0 + # via eg-feast (setup.py) +google-cloud-core==2.3.3 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.15.2 - # via feast (setup.py) +google-cloud-datastore==2.16.1 + # via eg-feast (setup.py) google-cloud-firestore==2.11.1 # via firebase-admin -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via - # feast (setup.py) + # eg-feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.5.0 @@ -295,21 +298,21 @@ google-resumable-media==2.5.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.59.0 +googleapis-common-protos[grpc]==1.60.0 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.15.50 - # via feast (setup.py) + # via eg-feast (setup.py) greenlet==2.0.2 # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.56.0 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -318,37 +321,38 @@ grpcio==1.54.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 - # via feast (setup.py) -grpcio-status==1.54.2 + # pymilvus +grpcio-reflection==1.56.0 + # via eg-feast (setup.py) +grpcio-status==1.56.0 # via google-api-core -grpcio-testing==1.54.2 - # via feast (setup.py) -grpcio-tools==1.54.2 - # via feast (setup.py) -gunicorn==20.1.0 - # via feast (setup.py) +grpcio-testing==1.56.0 + # via eg-feast (setup.py) +grpcio-tools==1.56.0 + # via eg-feast (setup.py) +gunicorn==21.2.0 + # via eg-feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) -hazelcast-python-client==5.2.0 - # via feast (setup.py) + # via eg-feast (setup.py) +hazelcast-python-client==5.3.0 + # via eg-feast (setup.py) hiredis==2.2.3 - # via feast (setup.py) -httpcore==0.17.2 + # via eg-feast (setup.py) +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 - # via feast (setup.py) -identify==2.5.24 + # via eg-feast (setup.py) +identify==2.5.26 # via pre-commit idna==3.4 # via @@ -360,27 +364,22 @@ idna==3.4 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via # dask # great-expectations iniconfig==2.0.0 # via pytest -ipykernel==6.23.1 +ipykernel==6.25.0 # via # ipywidgets - # nbclassic - # notebook + # jupyterlab ipython==8.14.0 # via # great-expectations # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipywidgets==8.0.6 +ipywidgets==8.0.7 # via great-expectations isodate==0.6.1 # via @@ -389,68 +388,81 @@ isodate==0.6.1 isoduration==20.11.0 # via jsonschema isort==5.12.0 - # via feast (setup.py) -jedi==0.18.2 + # via eg-feast (setup.py) +jedi==0.19.0 # via ipython jinja2==3.1.2 # via # altair - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # jupyter-server + # jupyterlab + # jupyterlab-server # moto - # nbclassic # nbconvert - # notebook # sphinx jmespath==1.0.1 # via # boto3 # botocore -jsonpatch==1.32 +json5==0.9.14 + # via jupyterlab-server +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.17.3 +jsonschema[format-nongpl]==4.18.4 # via # altair - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # jupyter-events + # jupyterlab-server # nbformat -jupyter-client==8.2.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.3.0 # via # ipykernel # jupyter-server - # nbclassic # nbclient - # notebook -jupyter-core==5.3.0 +jupyter-core==5.3.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # nbclient # nbconvert # nbformat - # notebook -jupyter-events==0.6.3 +jupyter-events==0.7.0 # via jupyter-server -jupyter-server==2.6.0 +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.0 # via - # nbclassic + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server +jupyterlab==4.0.3 + # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.7 +jupyterlab-server==2.24.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.8 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) + # via eg-feast (setup.py) locket==1.0.0 # via partd makefun==1.15.1 @@ -460,29 +472,33 @@ markupsafe==2.1.3 # jinja2 # nbconvert # werkzeug -marshmallow==3.19.0 - # via great-expectations +marshmallow==3.20.1 + # via + # environs + # great-expectations matplotlib-inline==0.1.6 # via # ipykernel # ipython mccabe==0.7.0 # via flake8 +milvus==2.2.12 + # via eg-feast (setup.py) minio==7.1.0 - # via feast (setup.py) -mistune==2.0.5 + # via eg-feast (setup.py) +mistune==3.0.1 # via # great-expectations # nbconvert -mmh3==4.0.0 - # via feast (setup.py) +mmh3==4.0.1 + # via eg-feast (setup.py) mock==2.0.0 - # via feast (setup.py) + # via eg-feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.1.10 - # via feast (setup.py) -msal==1.22.0 +moto==4.1.14 + # via eg-feast (setup.py) +msal==1.23.0 # via # azure-datalake-store # azure-identity @@ -499,53 +515,45 @@ multidict==6.0.4 # via # aiohttp # yarl -multiprocess==0.70.14 +multiprocess==0.70.15 # via bytewax mypy==0.982 # via - # feast (setup.py) + # eg-feast (setup.py) # sqlalchemy mypy-extensions==1.0.0 # via # black # mypy -mypy-protobuf==3.1 - # via feast (setup.py) -mysqlclient==2.1.1 - # via feast (setup.py) -nbclassic==1.0.0 - # via notebook +mypy-protobuf==3.1.0 + # via eg-feast (setup.py) +mysqlclient==2.2.0 + # via eg-feast (setup.py) nbclient==0.8.0 # via nbconvert -nbconvert==7.4.0 - # via - # jupyter-server - # nbclassic - # notebook -nbformat==5.9.0 +nbconvert==7.7.3 + # via jupyter-server +nbformat==5.9.2 # via # great-expectations # jupyter-server - # nbclassic # nbclient # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # ipykernel - # nbclassic - # notebook +nest-asyncio==1.5.7 + # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==6.5.4 +notebook==7.0.1 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.25.2 # via # altair # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # pandas # pandavro @@ -566,8 +574,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -578,20 +589,21 @@ pandas==1.5.3 # via # altair # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # great-expectations # pandavro + # pymilvus # snowflake-connector-python pandavro==1.5.2 - # via feast (setup.py) + # via eg-feast (setup.py) pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi partd==1.4.0 # via dask -pathspec==0.11.1 +pathspec==0.11.2 # via black pbr==5.11.1 # via mock @@ -600,38 +612,35 @@ pexpect==4.8.0 pickleshare==0.7.5 # via ipython pip-tools==7.1.0 - # via feast (setup.py) -platformdirs==3.5.1 + # via eg-feast (setup.py) +platformdirs==3.10.0 # via # black # jupyter-core # virtualenv -pluggy==1.0.0 +pluggy==1.2.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.7.0 # via msal-extensions -pre-commit==3.3.2 - # via feast (setup.py) -prometheus-client==0.17.0 - # via - # jupyter-server - # nbclassic - # notebook -prompt-toolkit==3.0.38 +pre-commit==3.3.3 + # via eg-feast (setup.py) +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 # via ipython -proto-plus==1.22.2 +proto-plus==1.22.3 # via - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.4 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -646,12 +655,13 @@ protobuf==4.23.2 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus psutil==5.9.0 # via - # feast (setup.py) + # eg-feast (setup.py) # ipykernel psycopg2-binary==2.9.6 - # via feast (setup.py) + # via eg-feast (setup.py) ptyprocess==0.7.0 # via # pexpect @@ -659,7 +669,7 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) + # via eg-feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 @@ -667,7 +677,7 @@ py4j==0.10.9.7 pyarrow==10.0.1 # via # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # snowflake-connector-python pyasn1==0.5.0 @@ -677,52 +687,52 @@ pyasn1==0.5.0 pyasn1-modules==0.3.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) -pycodestyle==2.10.0 + # via eg-feast (setup.py) +pycodestyle==2.11.0 # via flake8 pycparser==2.21 # via cffi pycryptodomex==3.18.0 # via snowflake-connector-python -pydantic==1.10.8 +pydantic==1.10.12 # via + # eg-feast (setup.py) # fastapi - # feast (setup.py) # great-expectations -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 pygments==2.15.1 # via - # feast (setup.py) + # eg-feast (setup.py) # ipython # nbconvert # sphinx -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via # adal # msal # snowflake-connector-python -pymssql==2.2.7 - # via feast (setup.py) -pymysql==1.0.3 - # via feast (setup.py) +pymilvus==2.2.14 + # via eg-feast (setup.py) +pymssql==2.2.8 + # via eg-feast (setup.py) +pymysql==1.1.0 + # via eg-feast (setup.py) pyodbc==4.0.39 - # via feast (setup.py) + # via eg-feast (setup.py) pyopenssl==23.2.0 # via snowflake-connector-python -pyparsing==3.0.9 +pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyspark==3.4.0 - # via feast (setup.py) -pytest==7.3.1 +pyspark==3.4.1 + # via eg-feast (setup.py) +pytest==7.4.0 # via - # feast (setup.py) + # eg-feast (setup.py) # pytest-benchmark # pytest-cov # pytest-lazy-fixture @@ -731,19 +741,19 @@ pytest==7.3.1 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-cov==4.1.0 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-xdist==3.3.1 - # via feast (setup.py) + # via eg-feast (setup.py) python-dateutil==2.8.2 # via # adal @@ -758,7 +768,9 @@ python-dateutil==2.8.2 # rockset # trino python-dotenv==1.0.0 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events pytz==2023.3 @@ -767,10 +779,10 @@ pytz==2023.3 # pandas # snowflake-connector-python # trino -pyyaml==6.0 +pyyaml==6.0.1 # via # dask - # feast (setup.py) + # eg-feast (setup.py) # jupyter-events # kubernetes # pre-commit @@ -781,12 +793,15 @@ pyzmq==25.1.0 # ipykernel # jupyter-client # jupyter-server - # nbclassic - # notebook redis==4.2.2 - # via feast (setup.py) -regex==2023.5.5 - # via feast (setup.py) + # via eg-feast (setup.py) +referencing==0.30.0 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.6.3 + # via eg-feast (setup.py) requests==2.31.0 # via # adal @@ -795,12 +810,13 @@ requests==2.31.0 # azure-datalake-store # cachecontrol # docker - # feast (setup.py) + # eg-feast (setup.py) # gcsfs # google-api-core # google-cloud-bigquery # google-cloud-storage # great-expectations + # jupyterlab-server # kubernetes # moto # msal @@ -825,21 +841,22 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.0.0 - # via feast (setup.py) +rockset==2.0.2 + # via eg-feast (setup.py) +rpds-py==0.9.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations s3transfer==0.6.1 # via boto3 -scipy==1.10.1 +scipy==1.11.1 # via great-expectations send2trash==1.8.2 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server six==1.16.0 # via # asttokens @@ -867,13 +884,13 @@ sniffio==1.3.0 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.0.4 - # via feast (setup.py) + # via eg-feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.4.1 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) + # via eg-feast (setup.py) sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -886,38 +903,36 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.48 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy[mypy]==1.4.49 + # via eg-feast (setup.py) +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy stack-data==0.6.2 # via ipython starlette==0.27.0 # via fastapi tabulate==0.9.0 - # via feast (setup.py) + # via eg-feast (setup.py) tenacity==8.2.2 - # via feast (setup.py) + # via eg-feast (setup.py) terminado==0.17.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic - # notebook testcontainers==3.7.1 - # via feast (setup.py) + # via eg-feast (setup.py) thriftpy2==0.4.16 # via happybase tinycss2==1.2.1 # via nbconvert toml==0.10.2 - # via feast (setup.py) + # via eg-feast (setup.py) tomli==2.0.1 # via - # black # build - # coverage + # jupyterlab # mypy + # pip-tools # pyproject-hooks # pytest toolz==0.12.0 @@ -930,12 +945,12 @@ tornado==6.3.2 # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # notebook # terminado tqdm==4.65.0 # via - # feast (setup.py) + # eg-feast (setup.py) # great-expectations traitlets==5.9.0 # via @@ -947,56 +962,60 @@ traitlets==5.9.0 # jupyter-core # jupyter-events # jupyter-server + # jupyterlab # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat - # notebook trino==0.326.0 - # via feast (setup.py) + # via eg-feast (setup.py) typeguard==2.13.3 - # via feast (setup.py) + # via eg-feast (setup.py) types-protobuf==3.19.22 # via - # feast (setup.py) + # eg-feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19.7 - # via feast (setup.py) -types-pyopenssl==23.2.0.0 +types-pymysql==1.1.0.1 + # via eg-feast (setup.py) +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 - # via feast (setup.py) +types-python-dateutil==2.8.19.14 + # via eg-feast (setup.py) types-pytz==2023.3.0.0 - # via feast (setup.py) -types-pyyaml==6.0.12.10 + # via eg-feast (setup.py) +types-pyyaml==6.0.12.11 # via - # feast (setup.py) + # eg-feast (setup.py) # responses -types-redis==4.5.5.2 - # via feast (setup.py) -types-requests==2.31.0.1 - # via feast (setup.py) -types-setuptools==67.8.0.0 - # via feast (setup.py) -types-tabulate==0.9.0.2 - # via feast (setup.py) -types-urllib3==1.26.25.13 +types-redis==4.6.0.3 + # via eg-feast (setup.py) +types-requests==2.31.0.2 + # via eg-feast (setup.py) +types-setuptools==68.0.0.3 + # via eg-feast (setup.py) +types-tabulate==0.9.0.3 + # via eg-feast (setup.py) +types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via + # async-lru # azure-core # azure-storage-blob + # fastapi # great-expectations # mypy # pydantic # snowflake-connector-python # sqlalchemy2-stubs + # uvicorn tzlocal==5.0.1 # via # great-expectations # trino -uri-template==1.2.0 +ujson==5.8.0 + # via pymilvus +uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client @@ -1004,7 +1023,7 @@ urllib3==1.26.16 # via # botocore # docker - # feast (setup.py) + # eg-feast (setup.py) # google-auth # great-expectations # kubernetes @@ -1013,11 +1032,11 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.22.0 - # via feast (setup.py) +uvicorn[standard]==0.23.2 + # via eg-feast (setup.py) uvloop==0.17.0 # via uvicorn -virtualenv==20.23.0 +virtualenv==20.24.2 # via pre-commit volatile==2.1.0 # via bowler @@ -1031,18 +1050,18 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.2 +websocket-client==1.6.1 # via # docker # jupyter-server # kubernetes websockets==11.0.3 # via uvicorn -werkzeug==2.3.4 +werkzeug==2.3.6 # via moto -wheel==0.40.0 +wheel==0.41.0 # via pip-tools -widgetsnbextension==4.0.7 +widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 # via @@ -1052,7 +1071,7 @@ xmltodict==0.13.0 # via moto yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 09e9485996..e12b97f004 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -311,7 +311,7 @@ greenlet==2.0.2 # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.56.0 # via # feast (setup.py) # google-api-core @@ -322,16 +322,17 @@ grpcio==1.54.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 - # via feast (setup.py) -grpcio-status==1.54.2 + # pymilvus +grpcio-reflection==1.56.0 + # via eg-feast (setup.py) +grpcio-status==1.56.0 # via google-api-core -grpcio-testing==1.54.2 - # via feast (setup.py) -grpcio-tools==1.54.2 - # via feast (setup.py) -gunicorn==20.1.0 - # via feast (setup.py) +grpcio-testing==1.56.0 + # via eg-feast (setup.py) +grpcio-tools==1.56.0 + # via eg-feast (setup.py) +gunicorn==21.2.0 + # via eg-feast (setup.py) h11==0.14.0 # via # httpcore @@ -477,6 +478,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +milvus==2.2.12 + # via eg-feast (setup.py) minio==7.1.0 # via feast (setup.py) mistune==2.0.5 @@ -549,16 +552,19 @@ nodeenv==1.8.0 notebook==6.5.4 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.24.4 # via # altair # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # pandas # pandavro # pyarrow + # pymilvus # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -575,8 +581,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -591,6 +600,7 @@ pandas==1.5.3 # google-cloud-bigquery # great-expectations # pandavro + # pymilvus # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) @@ -640,7 +650,7 @@ proto-plus==1.22.2 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -657,6 +667,7 @@ protobuf==4.23.2 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus psutil==5.9.0 # via # feast (setup.py) @@ -713,6 +724,8 @@ pyjwt[crypto]==2.7.0 # adal # msal # snowflake-connector-python +pymilvus==2.2.14 + # via eg-feast (setup.py) pymssql==2.2.7 # via feast (setup.py) pymysql==1.0.3 @@ -1013,6 +1026,8 @@ tzlocal==5.0.1 # via # great-expectations # trino +ujson==5.8.0 + # via pymilvus uri-template==1.2.0 # via jsonschema uritemplate==4.1.1 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e212808311..0abe3984c0 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # adal==1.2.7 # via msrestazure adlfs==0.5.9 - # via feast (setup.py) -aiohttp==3.8.4 + # via eg-feast (setup.py) +aiohttp==3.8.5 # via # adlfs # gcsfs @@ -18,7 +18,7 @@ alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations -anyio==3.7.0 +anyio==3.7.1 # via # httpcore # jupyter-server @@ -31,10 +31,7 @@ appnope==0.1.3 # ipykernel # ipython argon2-cffi==21.3.0 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 @@ -44,9 +41,11 @@ asn1crypto==1.5.1 # oscrypto # snowflake-connector-python assertpy==1.1 - # via feast (setup.py) + # via eg-feast (setup.py) asttokens==2.2.1 # via stack-data +async-lru==2.0.4 + # via jupyterlab async-timeout==4.0.2 # via # aiohttp @@ -56,9 +55,10 @@ attrs==23.1.0 # aiohttp # bowler # jsonschema + # referencing avro==1.10.0 - # via feast (setup.py) -azure-core==1.27.0 + # via eg-feast (setup.py) +azure-core==1.28.0 # via # adlfs # azure-identity @@ -69,45 +69,47 @@ azure-datalake-store==0.0.53 azure-identity==1.13.0 # via # adlfs - # feast (setup.py) -azure-storage-blob==12.16.0 + # eg-feast (setup.py) +azure-storage-blob==12.17.0 # via # adlfs - # feast (setup.py) + # eg-feast (setup.py) babel==2.12.1 - # via sphinx + # via + # jupyterlab-server + # sphinx backcall==0.2.0 # via ipython beautifulsoup4==4.12.2 # via nbconvert black==22.12.0 - # via feast (setup.py) + # via eg-feast (setup.py) bleach==6.0.0 # via nbconvert -boto3==1.26.146 +boto3==1.28.15 # via - # feast (setup.py) + # eg-feast (setup.py) # moto -botocore==1.29.146 +botocore==1.31.15 # via # boto3 # moto # s3transfer bowler==0.9.0 - # via feast (setup.py) + # via eg-feast (setup.py) build==0.10.0 # via - # feast (setup.py) + # eg-feast (setup.py) # pip-tools bytewax==0.15.1 - # via feast (setup.py) -cachecontrol==0.13.0 + # via eg-feast (setup.py) +cachecontrol==0.13.1 # via firebase-admin cachetools==5.3.1 # via google-auth -cassandra-driver==3.27.0 - # via feast (setup.py) -certifi==2023.5.7 +cassandra-driver==3.28.0 + # via eg-feast (setup.py) +certifi==2023.7.22 # via # httpcore # httpx @@ -124,17 +126,17 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests # snowflake-connector-python -click==8.1.3 +click==8.1.6 # via # black # bowler # dask - # feast (setup.py) + # eg-feast (setup.py) # geomet # great-expectations # moreorless @@ -144,7 +146,7 @@ cloudpickle==2.2.1 # via dask colorama==0.4.6 # via - # feast (setup.py) + # eg-feast (setup.py) # great-expectations comm==0.1.3 # via ipykernel @@ -155,8 +157,7 @@ cryptography==40.0.2 # adal # azure-identity # azure-storage-blob - # cassandra-driver - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # moto # msal @@ -165,8 +166,8 @@ cryptography==40.0.2 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.5.1 - # via feast (setup.py) +dask==2023.7.1 + # via eg-feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery debugpy==1.6.7 @@ -181,50 +182,52 @@ deprecated==1.2.14 # via redis deprecation==2.1.0 # via testcontainers -dill==0.3.6 +dill==0.3.7 # via # bytewax - # feast (setup.py) + # eg-feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via - # feast (setup.py) + # eg-feast (setup.py) # testcontainers docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.1 +environs==9.5.0 + # via pymilvus +exceptiongroup==1.1.2 # via # anyio # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist executing==1.2.0 # via stack-data -fastapi==0.95.2 - # via feast (setup.py) -fastavro==1.7.4 +fastapi==0.100.1 + # via eg-feast (setup.py) +fastavro==1.8.2 # via - # feast (setup.py) + # eg-feast (setup.py) # pandavro -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 # via nbformat -filelock==3.12.0 +filelock==3.12.2 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) + # via eg-feast (setup.py) fissix==21.11.13 # via bowler -flake8==6.0.0 - # via feast (setup.py) +flake8==6.1.0 + # via eg-feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -234,14 +237,14 @@ fsspec==2022.1.0 # dask # gcsfs gcsfs==2022.1.0 - # via feast (setup.py) + # via eg-feast (setup.py) geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.0 +google-api-core[grpc]==2.11.1 # via - # feast (setup.py) + # eg-feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -251,9 +254,9 @@ google-api-core[grpc]==2.11.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.88.0 +google-api-python-client==2.95.0 # via firebase-admin -google-auth==2.19.1 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -267,26 +270,26 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-bigquery[pandas]==3.11.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.20.0 - # via feast (setup.py) -google-cloud-bigtable==2.18.1 - # via feast (setup.py) -google-cloud-core==2.3.2 +google-cloud-bigquery[pandas]==3.11.4 + # via eg-feast (setup.py) +google-cloud-bigquery-storage==2.22.0 + # via eg-feast (setup.py) +google-cloud-bigtable==2.20.0 + # via eg-feast (setup.py) +google-cloud-core==2.3.3 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.15.2 - # via feast (setup.py) +google-cloud-datastore==2.16.1 + # via eg-feast (setup.py) google-cloud-firestore==2.11.1 # via firebase-admin -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via - # feast (setup.py) + # eg-feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.5.0 @@ -295,21 +298,21 @@ google-resumable-media==2.5.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.59.0 +googleapis-common-protos[grpc]==1.59.1 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.15.50 - # via feast (setup.py) + # via eg-feast (setup.py) greenlet==2.0.2 # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.56.0 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -318,37 +321,38 @@ grpcio==1.54.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 - # via feast (setup.py) -grpcio-status==1.54.2 + # pymilvus +grpcio-reflection==1.56.0 + # via eg-feast (setup.py) +grpcio-status==1.56.0 # via google-api-core -grpcio-testing==1.54.2 - # via feast (setup.py) -grpcio-tools==1.54.2 - # via feast (setup.py) -gunicorn==20.1.0 - # via feast (setup.py) +grpcio-testing==1.56.0 + # via eg-feast (setup.py) +grpcio-tools==1.56.0 + # via eg-feast (setup.py) +gunicorn==21.2.0 + # via eg-feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) -hazelcast-python-client==5.2.0 - # via feast (setup.py) + # via eg-feast (setup.py) +hazelcast-python-client==5.3.0 + # via eg-feast (setup.py) hiredis==2.2.3 - # via feast (setup.py) -httpcore==0.17.2 + # via eg-feast (setup.py) +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 - # via feast (setup.py) -identify==2.5.24 + # via eg-feast (setup.py) +identify==2.5.26 # via pre-commit idna==3.4 # via @@ -360,30 +364,28 @@ idna==3.4 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via # dask # great-expectations # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server # nbconvert # sphinx iniconfig==2.0.0 # via pytest -ipykernel==6.23.1 +ipykernel==6.25.0 # via # ipywidgets - # nbclassic - # notebook + # jupyterlab ipython==8.14.0 # via # great-expectations # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipywidgets==8.0.6 +ipywidgets==8.0.7 # via great-expectations isodate==0.6.1 # via @@ -392,68 +394,81 @@ isodate==0.6.1 isoduration==20.11.0 # via jsonschema isort==5.12.0 - # via feast (setup.py) -jedi==0.18.2 + # via eg-feast (setup.py) +jedi==0.19.0 # via ipython jinja2==3.1.2 # via # altair - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # jupyter-server + # jupyterlab + # jupyterlab-server # moto - # nbclassic # nbconvert - # notebook # sphinx jmespath==1.0.1 # via # boto3 # botocore -jsonpatch==1.32 +json5==0.9.14 + # via jupyterlab-server +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.17.3 +jsonschema[format-nongpl]==4.18.4 # via # altair - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # jupyter-events + # jupyterlab-server # nbformat -jupyter-client==8.2.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.3.0 # via # ipykernel # jupyter-server - # nbclassic # nbclient - # notebook -jupyter-core==5.3.0 +jupyter-core==5.3.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # nbclient # nbconvert # nbformat - # notebook jupyter-events==0.6.3 # via jupyter-server -jupyter-server==2.6.0 +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.0 # via - # nbclassic + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server +jupyterlab==4.0.3 + # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.7 +jupyterlab-server==2.24.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.8 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) + # via eg-feast (setup.py) locket==1.0.0 # via partd makefun==1.15.1 @@ -463,29 +478,33 @@ markupsafe==2.1.3 # jinja2 # nbconvert # werkzeug -marshmallow==3.19.0 - # via great-expectations +marshmallow==3.20.1 + # via + # environs + # great-expectations matplotlib-inline==0.1.6 # via # ipykernel # ipython mccabe==0.7.0 # via flake8 +milvus==2.2.12 + # via eg-feast (setup.py) minio==7.1.0 - # via feast (setup.py) -mistune==2.0.5 + # via eg-feast (setup.py) +mistune==3.0.1 # via # great-expectations # nbconvert -mmh3==4.0.0 - # via feast (setup.py) +mmh3==4.0.1 + # via eg-feast (setup.py) mock==2.0.0 - # via feast (setup.py) + # via eg-feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.1.10 - # via feast (setup.py) -msal==1.22.0 +moto==4.1.14 + # via eg-feast (setup.py) +msal==1.23.0 # via # azure-datalake-store # azure-identity @@ -502,53 +521,45 @@ multidict==6.0.4 # via # aiohttp # yarl -multiprocess==0.70.14 +multiprocess==0.70.15 # via bytewax mypy==0.982 # via - # feast (setup.py) + # eg-feast (setup.py) # sqlalchemy mypy-extensions==1.0.0 # via # black # mypy -mypy-protobuf==3.1 - # via feast (setup.py) -mysqlclient==2.1.1 - # via feast (setup.py) -nbclassic==1.0.0 - # via notebook +mypy-protobuf==3.1.0 + # via eg-feast (setup.py) +mysqlclient==2.2.0 + # via eg-feast (setup.py) nbclient==0.8.0 # via nbconvert -nbconvert==7.4.0 - # via - # jupyter-server - # nbclassic - # notebook -nbformat==5.9.0 +nbconvert==7.7.3 + # via jupyter-server +nbformat==5.9.1 # via # great-expectations # jupyter-server - # nbclassic # nbclient # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # ipykernel - # nbclassic - # notebook +nest-asyncio==1.5.7 + # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==6.5.4 +notebook==7.0.0 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.25.1 # via # altair # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # great-expectations # pandas # pandavro @@ -569,8 +580,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -581,20 +595,21 @@ pandas==1.5.3 # via # altair # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # great-expectations # pandavro + # pymilvus # snowflake-connector-python pandavro==1.5.2 - # via feast (setup.py) + # via eg-feast (setup.py) pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi partd==1.4.0 # via dask -pathspec==0.11.1 +pathspec==0.11.2 # via black pbr==5.11.1 # via mock @@ -603,38 +618,35 @@ pexpect==4.8.0 pickleshare==0.7.5 # via ipython pip-tools==7.1.0 - # via feast (setup.py) -platformdirs==3.5.1 + # via eg-feast (setup.py) +platformdirs==3.10.0 # via # black # jupyter-core # virtualenv -pluggy==1.0.0 +pluggy==1.2.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.7.0 # via msal-extensions -pre-commit==3.3.2 - # via feast (setup.py) -prometheus-client==0.17.0 - # via - # jupyter-server - # nbclassic - # notebook -prompt-toolkit==3.0.38 +pre-commit==3.3.3 + # via eg-feast (setup.py) +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 # via ipython -proto-plus==1.22.2 +proto-plus==1.22.3 # via - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.4 # via - # feast (setup.py) + # eg-feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -649,12 +661,13 @@ protobuf==4.23.2 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus psutil==5.9.0 # via - # feast (setup.py) + # eg-feast (setup.py) # ipykernel psycopg2-binary==2.9.6 - # via feast (setup.py) + # via eg-feast (setup.py) ptyprocess==0.7.0 # via # pexpect @@ -662,7 +675,7 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) + # via eg-feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 @@ -670,7 +683,7 @@ py4j==0.10.9.7 pyarrow==10.0.1 # via # db-dtypes - # feast (setup.py) + # eg-feast (setup.py) # google-cloud-bigquery # snowflake-connector-python pyasn1==0.5.0 @@ -680,52 +693,52 @@ pyasn1==0.5.0 pyasn1-modules==0.3.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) -pycodestyle==2.10.0 + # via eg-feast (setup.py) +pycodestyle==2.11.0 # via flake8 pycparser==2.21 # via cffi pycryptodomex==3.18.0 # via snowflake-connector-python -pydantic==1.10.8 +pydantic==1.10.12 # via + # eg-feast (setup.py) # fastapi - # feast (setup.py) # great-expectations -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 pygments==2.15.1 # via - # feast (setup.py) + # eg-feast (setup.py) # ipython # nbconvert # sphinx -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via # adal # msal # snowflake-connector-python -pymssql==2.2.7 - # via feast (setup.py) -pymysql==1.0.3 - # via feast (setup.py) +pymilvus==2.2.14 + # via eg-feast (setup.py) +pymssql==2.2.8 + # via eg-feast (setup.py) +pymysql==1.1.0 + # via eg-feast (setup.py) pyodbc==4.0.39 - # via feast (setup.py) + # via eg-feast (setup.py) pyopenssl==23.2.0 # via snowflake-connector-python -pyparsing==3.0.9 +pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyspark==3.4.0 - # via feast (setup.py) -pytest==7.3.1 +pyspark==3.4.1 + # via eg-feast (setup.py) +pytest==7.4.0 # via - # feast (setup.py) + # eg-feast (setup.py) # pytest-benchmark # pytest-cov # pytest-lazy-fixture @@ -734,19 +747,19 @@ pytest==7.3.1 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-cov==4.1.0 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) + # via eg-feast (setup.py) pytest-xdist==3.3.1 - # via feast (setup.py) + # via eg-feast (setup.py) python-dateutil==2.8.2 # via # adal @@ -761,7 +774,9 @@ python-dateutil==2.8.2 # rockset # trino python-dotenv==1.0.0 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events pytz==2023.3 @@ -770,10 +785,10 @@ pytz==2023.3 # pandas # snowflake-connector-python # trino -pyyaml==6.0 +pyyaml==6.0.1 # via # dask - # feast (setup.py) + # eg-feast (setup.py) # jupyter-events # kubernetes # pre-commit @@ -784,12 +799,14 @@ pyzmq==25.1.0 # ipykernel # jupyter-client # jupyter-server - # nbclassic - # notebook redis==4.2.2 - # via feast (setup.py) -regex==2023.5.5 - # via feast (setup.py) + # via eg-feast (setup.py) +referencing==0.30.0 + # via + # jsonschema + # jsonschema-specifications +regex==2023.6.3 + # via eg-feast (setup.py) requests==2.31.0 # via # adal @@ -798,12 +815,13 @@ requests==2.31.0 # azure-datalake-store # cachecontrol # docker - # feast (setup.py) + # eg-feast (setup.py) # gcsfs # google-api-core # google-cloud-bigquery # google-cloud-storage # great-expectations + # jupyterlab-server # kubernetes # moto # msal @@ -828,8 +846,12 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.0.0 - # via feast (setup.py) +rockset==2.0.2 + # via eg-feast (setup.py) +rpds-py==0.9.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.17.17 @@ -838,13 +860,10 @@ ruamel-yaml-clib==0.2.7 # via ruamel-yaml s3transfer==0.6.1 # via boto3 -scipy==1.10.1 +scipy==1.11.1 # via great-expectations send2trash==1.8.2 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server six==1.16.0 # via # asttokens @@ -872,13 +891,13 @@ sniffio==1.3.0 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.0.4 - # via feast (setup.py) + # via eg-feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.4.1 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) + # via eg-feast (setup.py) sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -891,38 +910,38 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.48 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy[mypy]==1.4.49 + # via eg-feast (setup.py) +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy stack-data==0.6.2 # via ipython starlette==0.27.0 # via fastapi tabulate==0.9.0 - # via feast (setup.py) + # via eg-feast (setup.py) tenacity==8.2.2 - # via feast (setup.py) + # via eg-feast (setup.py) terminado==0.17.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic - # notebook testcontainers==3.7.1 - # via feast (setup.py) + # via eg-feast (setup.py) thriftpy2==0.4.16 # via happybase tinycss2==1.2.1 # via nbconvert toml==0.10.2 - # via feast (setup.py) + # via eg-feast (setup.py) tomli==2.0.1 # via # black # build # coverage + # jupyterlab # mypy + # pip-tools # pyproject-hooks # pytest toolz==0.12.0 @@ -935,12 +954,12 @@ tornado==6.3.2 # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # notebook # terminado tqdm==4.65.0 # via - # feast (setup.py) + # eg-feast (setup.py) # great-expectations traitlets==5.9.0 # via @@ -952,47 +971,48 @@ traitlets==5.9.0 # jupyter-core # jupyter-events # jupyter-server + # jupyterlab # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat - # notebook trino==0.326.0 - # via feast (setup.py) + # via eg-feast (setup.py) typeguard==2.13.3 - # via feast (setup.py) + # via eg-feast (setup.py) types-protobuf==3.19.22 # via - # feast (setup.py) + # eg-feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19.7 - # via feast (setup.py) -types-pyopenssl==23.2.0.0 +types-pymysql==1.1.0.1 + # via eg-feast (setup.py) +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 - # via feast (setup.py) +types-python-dateutil==2.8.19.14 + # via eg-feast (setup.py) types-pytz==2023.3.0.0 - # via feast (setup.py) -types-pyyaml==6.0.12.10 + # via eg-feast (setup.py) +types-pyyaml==6.0.12.11 # via - # feast (setup.py) + # eg-feast (setup.py) # responses -types-redis==4.5.5.2 - # via feast (setup.py) -types-requests==2.31.0.1 - # via feast (setup.py) -types-setuptools==67.8.0.0 - # via feast (setup.py) -types-tabulate==0.9.0.2 - # via feast (setup.py) -types-urllib3==1.26.25.13 +types-redis==4.6.0.3 + # via eg-feast (setup.py) +types-requests==2.31.0.2 + # via eg-feast (setup.py) +types-setuptools==68.0.0.3 + # via eg-feast (setup.py) +types-tabulate==0.9.0.3 + # via eg-feast (setup.py) +types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via + # async-lru # azure-core # azure-storage-blob # black + # fastapi # great-expectations # ipython # mypy @@ -1000,11 +1020,14 @@ typing-extensions==4.6.3 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # uvicorn tzlocal==5.0.1 # via # great-expectations # trino -uri-template==1.2.0 +ujson==5.8.0 + # via pymilvus +uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client @@ -1012,7 +1035,7 @@ urllib3==1.26.16 # via # botocore # docker - # feast (setup.py) + # eg-feast (setup.py) # google-auth # great-expectations # kubernetes @@ -1021,11 +1044,11 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.22.0 - # via feast (setup.py) +uvicorn[standard]==0.23.1 + # via eg-feast (setup.py) uvloop==0.17.0 # via uvicorn -virtualenv==20.23.0 +virtualenv==20.24.2 # via pre-commit volatile==2.1.0 # via bowler @@ -1039,18 +1062,18 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.2 +websocket-client==1.6.1 # via # docker # jupyter-server # kubernetes websockets==11.0.3 # via uvicorn -werkzeug==2.3.4 +werkzeug==2.3.6 # via moto -wheel==0.40.0 +wheel==0.41.0 # via pip-tools -widgetsnbextension==4.0.7 +widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 # via @@ -1060,7 +1083,7 @@ xmltodict==0.13.0 # via moto yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/tests/expediagroup/milvus_online_store_creator.py b/sdk/python/tests/expediagroup/milvus_online_store_creator.py new file mode 100644 index 0000000000..ab21761ead --- /dev/null +++ b/sdk/python/tests/expediagroup/milvus_online_store_creator.py @@ -0,0 +1,24 @@ +from typing import Dict + +from milvus import default_server + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MilvusOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.host = "localhost" + self.port = 19530 + self.server = default_server + self.server.wait_for_started = False + + def create_online_store(self) -> Dict[str, str]: + self.server.start() + + return {"type": "milvus", "host": self.host, "port": str(self.port)} + + def teardown(self): + self.server.stop() diff --git a/sdk/python/tests/expediagroup/test_milvus_online_store.py b/sdk/python/tests/expediagroup/test_milvus_online_store.py new file mode 100644 index 0000000000..f15d2c1f0f --- /dev/null +++ b/sdk/python/tests/expediagroup/test_milvus_online_store.py @@ -0,0 +1,37 @@ +from pymilvus.client.stub import Milvus +from pymilvus.client.types import DataType + +from tests.expediagroup.milvus_online_store_creator import MilvusOnlineStoreCreator + + +def test_milvus_start_stop(): + # this is just an example how to start / stop Milvus. Once a real test is implemented this test can be deleted + online_store_creator = MilvusOnlineStoreCreator("milvus") + online_store_creator.create_online_store() + + # access through a stub + milvus = Milvus(online_store_creator.host, 19530) + + # Create collection demo_collection if it doesn't exist. + collection_name = "example_collection_" + + ok = milvus.has_collection(collection_name) + if not ok: + fields = { + "fields": [ + {"name": "key", "type": DataType.INT64, "is_primary": True}, + { + "name": "vector", + "type": DataType.FLOAT_VECTOR, + "params": {"dim": 32}, + }, + ], + "auto_id": True, + } + + milvus.create_collection(collection_name, fields) + + collection = milvus.describe_collection(collection_name) + assert collection.get("collection_name") == collection_name + + online_store_creator.teardown() diff --git a/setup.py b/setup.py index 5ab3d3903c..6dcfff46f1 100644 --- a/setup.py +++ b/setup.py @@ -138,6 +138,11 @@ "hazelcast-python-client>=5.1", ] +MILVUS_REQUIRED = [ + "milvus==2.2.12", + "pymilvus==2.2.14" +] + CI_REQUIRED = ( [ "build", @@ -197,6 +202,7 @@ + AZURE_REQUIRED + ROCKSET_REQUIRED + HAZELCAST_REQUIRED + + MILVUS_REQUIRED ) @@ -364,6 +370,7 @@ def run(self): "cassandra": CASSANDRA_REQUIRED, "hazelcast": HAZELCAST_REQUIRED, "rockset": ROCKSET_REQUIRED, + "milvus": MILVUS_REQUIRED, }, include_package_data=True, license="Apache",