From c8fae699f9c90f46e5036ae6744869e4dc3598db Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 11:37:40 +0200 Subject: [PATCH 01/20] Makefile: Formatting Signed-off-by: Job Almekinders --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2ad693c7a1..552cb1a7b7 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ install-python: python setup.py develop lock-python-dependencies: - uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt + uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt lock-python-dependencies-all: pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt" @@ -164,7 +164,7 @@ test-python-universal-mssql: sdk/python/tests -# To use Athena as an offline store, you need to create an Athena database and an S3 bucket on AWS. +# To use Athena as an offline store, you need to create an Athena database and an S3 bucket on AWS. # https://docs.aws.amazon.com/athena/latest/ug/getting-started.html # Modify environment variables ATHENA_REGION, ATHENA_DATA_SOURCE, ATHENA_DATABASE, ATHENA_WORKGROUP or # ATHENA_S3_BUCKET_NAME according to your needs. If tests fail with the pytest -n 8 option, change the number to 1. @@ -191,7 +191,7 @@ test-python-universal-athena: not s3_registry and \ not test_snowflake" \ sdk/python/tests - + test-python-universal-postgres-offline: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \ From 161fd9d0ca8f94af23dd35a136007de7e21e2d16 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 11:38:21 +0200 Subject: [PATCH 02/20] Makefile: Exclude Snowflake tests for postgres offline store tests Signed-off-by: Job Almekinders --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 552cb1a7b7..68a107288c 100644 --- a/Makefile +++ b/Makefile @@ -209,6 +209,7 @@ test-python-universal-postgres-offline: not test_push_features_to_offline_store and \ not gcs_registry and \ not s3_registry and \ + not test_snowflake and \ not test_universal_types" \ sdk/python/tests From df0f440ebf711ba554b5f37c24d68410905372fb Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 11:43:31 +0200 Subject: [PATCH 03/20] Requirements: Update Signed-off-by: Job Almekinders --- .../requirements/py3.10-ci-requirements.txt | 157 ++++++++++++++++-- .../requirements/py3.10-requirements.txt | 48 +++++- .../requirements/py3.11-ci-requirements.txt | 157 ++++++++++++++++-- .../requirements/py3.11-requirements.txt | 48 +++++- .../requirements/py3.9-ci-requirements.txt | 157 ++++++++++++++++-- .../requirements/py3.9-requirements.txt | 48 +++++- setup.py | 2 +- 7 files changed, 535 insertions(+), 82 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 97bdfc159b..02b9da1f61 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.13.0 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -48,7 +52,9 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -60,7 +66,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.99 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.99 # via # aiobotocore @@ -68,12 +76,15 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # feast (setup.py) + # pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via # elastic-transport @@ -96,6 +107,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -105,7 +117,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -114,6 +128,7 @@ coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -125,7 +140,9 @@ cryptography==42.0.7 # types-pyopenssl # types-redis dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask db-dtypes==1.2.0 @@ -137,7 +154,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.17.3 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -155,6 +174,7 @@ duckdb-engine==0.12.1 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -169,7 +189,9 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fastjsonschema==2.19.1 @@ -179,6 +201,7 @@ filelock==3.14.0 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -186,13 +209,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.0 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -217,8 +243,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.23.1 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -227,10 +256,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -241,16 +273,17 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable grpcio==1.64.0 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -261,19 +294,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -284,11 +325,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -323,6 +368,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -346,6 +392,7 @@ jsonpointer==2.4 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -391,6 +438,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -411,13 +459,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -433,10 +485,13 @@ multidict==6.0.5 multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -459,6 +514,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -495,6 +551,7 @@ packaging==24.0 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -518,6 +575,7 @@ pexpect==4.9.0 pip==24.0 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -530,6 +588,7 @@ ply==3.11 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.45 @@ -544,6 +603,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -561,8 +621,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -570,12 +637,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -593,16 +662,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -612,8 +684,11 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -625,8 +700,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -636,13 +713,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -671,6 +756,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -684,14 +770,17 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.4.28 + # via feast (setup.py) requests==2.31.0 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -725,6 +814,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -734,6 +824,7 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.6 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.13.1 @@ -767,11 +858,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.10.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -786,6 +879,7 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.30 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -794,6 +888,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -801,17 +896,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.0 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -838,7 +937,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -855,25 +956,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240425 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.0.0.20240524 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 @@ -887,6 +1002,8 @@ typing-extensions==4.11.0 # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -909,6 +1026,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.18 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -921,12 +1039,15 @@ urllib3==1.26.18 # testcontainers uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 99c9bfc3fe..72124636b6 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -65,8 +71,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -78,13 +87,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -96,20 +108,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -120,6 +141,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -127,6 +149,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -142,11 +165,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -154,7 +181,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -178,6 +207,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index f6db0af6bc..ad824786f2 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.13.0 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -44,7 +48,9 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -56,7 +62,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.99 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.99 # via # aiobotocore @@ -64,12 +72,15 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # feast (setup.py) + # pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via # elastic-transport @@ -92,6 +103,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -101,7 +113,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -110,6 +124,7 @@ coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -121,7 +136,9 @@ cryptography==42.0.7 # types-pyopenssl # types-redis dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask db-dtypes==1.2.0 @@ -133,7 +150,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.17.4 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -151,6 +170,7 @@ duckdb-engine==0.12.1 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -160,7 +180,9 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fastjsonschema==2.19.1 @@ -170,6 +192,7 @@ filelock==3.14.0 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -177,13 +200,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.0 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -208,8 +234,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.23.1 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -218,10 +247,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -232,16 +264,17 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable grpcio==1.64.0 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -252,19 +285,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -275,11 +316,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -314,6 +359,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -337,6 +383,7 @@ jsonpointer==2.4 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -382,6 +429,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -402,13 +450,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -424,10 +476,13 @@ multidict==6.0.5 multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -450,6 +505,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -486,6 +542,7 @@ packaging==24.0 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -509,6 +566,7 @@ pexpect==4.9.0 pip==24.0 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -521,6 +579,7 @@ ply==3.11 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.45 @@ -535,6 +594,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -552,8 +612,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -561,12 +628,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -584,16 +653,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -603,8 +675,11 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -616,8 +691,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -627,13 +704,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -662,6 +747,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -675,14 +761,17 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 + # via feast (setup.py) requests==2.31.0 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -716,6 +805,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -725,6 +815,7 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.6 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.13.1 @@ -758,11 +849,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.10.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -777,6 +870,7 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.30 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -785,6 +879,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -792,17 +887,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.0 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -820,7 +919,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -837,25 +938,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240425 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.0.0.20240524 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 @@ -867,6 +982,8 @@ typing-extensions==4.11.0 # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -888,6 +1005,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.18 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -900,12 +1018,15 @@ urllib3==1.26.18 # testcontainers uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c34b610d14..408f392515 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -20,30 +20,36 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -63,8 +69,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -76,13 +85,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -94,20 +106,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -118,6 +139,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -125,6 +147,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -140,17 +163,23 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) toolz==0.12.1 # via # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -172,6 +201,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 135b65a0cc..e159a45c6a 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.13.0 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -48,7 +52,9 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -60,7 +66,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.99 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.99 # via # aiobotocore @@ -68,12 +76,15 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # feast (setup.py) + # pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via # elastic-transport @@ -96,6 +107,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -105,7 +117,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -114,6 +128,7 @@ coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -125,7 +140,9 @@ cryptography==42.0.7 # types-pyopenssl # types-redis dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask db-dtypes==1.2.0 @@ -137,7 +154,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.17.4 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -155,6 +174,7 @@ duckdb-engine==0.12.1 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -169,7 +189,9 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fastjsonschema==2.19.1 @@ -179,6 +201,7 @@ filelock==3.14.0 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -186,13 +209,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.0 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -217,8 +243,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.23.1 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -227,10 +256,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -241,16 +273,17 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable grpcio==1.64.0 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -261,19 +294,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -284,11 +325,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -332,6 +377,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -355,6 +401,7 @@ jsonpointer==2.4 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -400,6 +447,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -420,13 +468,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -442,10 +494,13 @@ multidict==6.0.5 multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -468,6 +523,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -504,6 +560,7 @@ packaging==24.0 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -527,6 +584,7 @@ pexpect==4.9.0 pip==24.0 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -539,6 +597,7 @@ ply==3.11 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.45 @@ -553,6 +612,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -570,8 +630,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.18 + # via feast (setup.py) +psycopg-binary==3.1.18 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -579,12 +646,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -602,16 +671,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -621,8 +693,11 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -634,8 +709,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -645,13 +722,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -680,6 +765,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -693,14 +779,17 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 + # via feast (setup.py) requests==2.31.0 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -734,6 +823,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -745,6 +835,7 @@ ruamel-yaml==0.17.17 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.4.6 + # via feast (setup.py) s3transfer==0.10.1 # via boto3 scipy==1.13.1 @@ -778,11 +869,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.10.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -797,6 +890,7 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.30 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -805,6 +899,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -812,17 +907,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.0 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -849,7 +948,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -866,25 +967,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.0.0.20240524 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 @@ -899,6 +1014,8 @@ typing-extensions==4.11.0 # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -922,6 +1039,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.18 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -935,12 +1053,15 @@ urllib3==1.26.18 # testcontainers uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 149a96626e..3c833438de 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -67,8 +73,11 @@ importlib-metadata==7.1.0 # dask # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -80,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -98,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -122,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -129,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -144,11 +167,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -156,7 +183,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -181,6 +210,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/setup.py b/setup.py index f954f19898..fbebfb643f 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] POSTGRES_REQUIRED = [ - "psycopg2-binary>=2.8.3,<3", + "psycopg[binary,pool]>=3.0.0,<4", ] MYSQL_REQUIRED = ["pymysql", "types-PyMySQL"] From c20025413bd88fe4907a22ba31c6c0f86653fee3 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 13:22:04 +0200 Subject: [PATCH 04/20] Bootstrap: Use conninfo Signed-off-by: Job Almekinders --- sdk/python/feast/templates/postgres/bootstrap.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/templates/postgres/bootstrap.py b/sdk/python/feast/templates/postgres/bootstrap.py index 9f6e8a988d..6ed13e4e39 100644 --- a/sdk/python/feast/templates/postgres/bootstrap.py +++ b/sdk/python/feast/templates/postgres/bootstrap.py @@ -1,5 +1,5 @@ import click -import psycopg2 +import psycopg from feast.file_utils import replace_str_in_file from feast.infra.utils.postgres.connection_utils import df_to_postgres_table @@ -34,12 +34,14 @@ def bootstrap(): 'Should I upload example data to Postgres (overwriting "feast_driver_hourly_stats" table)?', default=True, ): - db_connection = psycopg2.connect( - dbname=postgres_database, - host=postgres_host, - port=int(postgres_port), - user=postgres_user, - password=postgres_password, + db_connection = psycopg.connect( + conninfo=( + f"postgresql://{postgres_user}" + f":{postgres_password}" + f"@{postgres_host}" + f":{int(postgres_port)}" + f"/{postgres_database}" + ), options=f"-c search_path={postgres_schema}", ) From 861857929914ed11c6b074961c615fa7ef5a3850 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 13:41:28 +0200 Subject: [PATCH 05/20] Tests: Make connection string compatible with psycopg3 Signed-off-by: Job Almekinders --- .../tests/integration/registration/test_universal_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 24ba9fe42a..27323e6515 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -149,7 +149,7 @@ def pg_registry(): registry_config = RegistryConfig( registry_type="sql", - path=f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + path=f"postgresql+psycopg://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) From 86c22e5094407da73842546e9154861946ed640f Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 17:33:49 +0200 Subject: [PATCH 06/20] Tests: Test connection type pool and singleton Signed-off-by: Job Almekinders --- .../integration/online_store/test_universal_online.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index e78c1053bf..c6b034e2aa 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -39,13 +39,18 @@ @pytest.mark.integration @pytest.mark.universal_online_stores(only=["postgres"]) +@pytest.mark.parametrize( + "conn_type", + [ConnectionType.singleton, ConnectionType.pool], + ids=lambda v: f"conn_type:{v}", +) def test_connection_pool_online_stores( - environment, universal_data_sources, fake_ingest_data + environment, universal_data_sources, fake_ingest_data, conn_type ): if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": return fs = environment.feature_store - fs.config.online_store.conn_type = ConnectionType.pool + fs.config.online_store.conn_type = conn_type fs.config.online_store.min_conn = 1 fs.config.online_store.max_conn = 10 From 8fb922482e116713e7be56f09eddf3fcc522e39b Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 10:36:42 +0200 Subject: [PATCH 07/20] Global: Replace conn.set_session() calls to be psycopg3 compatible Set connection read only Signed-off-by: Job Almekinders --- .../offline_stores/contrib/postgres_offline_store/postgres.py | 2 +- sdk/python/feast/infra/utils/postgres/connection_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index cb08b5f016..0f859d8e3d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -274,7 +274,7 @@ def to_sql(self) -> str: def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: with self._query_generator() as query: with _get_conn(self.config.offline_store) as conn, conn.cursor() as cur: - conn.set_session(readonly=True) + conn.read_only = True cur.execute(query) fields = [ (c.name, pg_type_code_to_arrow(c.type_code)) diff --git a/sdk/python/feast/infra/utils/postgres/connection_utils.py b/sdk/python/feast/infra/utils/postgres/connection_utils.py index 0d99c8ab99..3ffc1866bf 100644 --- a/sdk/python/feast/infra/utils/postgres/connection_utils.py +++ b/sdk/python/feast/infra/utils/postgres/connection_utils.py @@ -82,7 +82,7 @@ def get_query_schema(config: PostgreSQLConfig, sql_query: str) -> Dict[str, np.d new table """ with _get_conn(config) as conn: - conn.set_session(readonly=True) + conn.read_only = True df = pd.read_sql( f"SELECT * FROM {sql_query} LIMIT 0", conn, From 0393d6dec1bf6fdde88a0efcd968c8d450b557c6 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 12:41:01 +0200 Subject: [PATCH 08/20] Offline: Use psycopg3 Signed-off-by: Job Almekinders --- .../offline_stores/contrib/postgres_offline_store/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index 0f859d8e3d..6a8686d282 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -19,7 +19,7 @@ import pandas as pd import pyarrow as pa from jinja2 import BaseLoader, Environment -from psycopg2 import sql +from psycopg import sql from pytz import utc from feast.data_source import DataSource From 2a7ad9a52712553419903a8d628b0e8453fb4e85 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 16:23:10 +0200 Subject: [PATCH 09/20] Online: Use psycopg3 Signed-off-by: Job Almekinders --- .../feast/infra/online_stores/contrib/postgres.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 3eddd8ba20..623637acd5 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -4,11 +4,10 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple -import psycopg2 import pytz -from psycopg2 import sql -from psycopg2.extras import execute_values -from psycopg2.pool import SimpleConnectionPool +from psycopg import sql +from psycopg.connection import Connection +from psycopg_pool import ConnectionPool from feast import Entity from feast.feature_view import FeatureView @@ -39,8 +38,8 @@ class PostgreSQLOnlineStoreConfig(PostgreSQLConfig): class PostgreSQLOnlineStore(OnlineStore): - _conn: Optional[psycopg2._psycopg.connection] = None - _conn_pool: Optional[SimpleConnectionPool] = None + _conn: Optional[Connection] = None + _conn_pool: Optional[ConnectionPool] = None @contextlib.contextmanager def _get_conn(self, config: RepoConfig): @@ -96,8 +95,7 @@ def online_write_batch( batch_size = 5000 for i in range(0, len(insert_values), batch_size): cur_batch = insert_values[i : i + batch_size] - execute_values( - cur, + cur.executemany( sql.SQL( """ INSERT INTO {} @@ -112,7 +110,6 @@ def online_write_batch( """, ).format(sql.Identifier(_table_id(project, table))), cur_batch, - page_size=batch_size, ) conn.commit() if progress: From 995fc085a466710f1d31791827c0b8368a881915 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 16:03:52 +0200 Subject: [PATCH 10/20] Online: Restructure online_write_batch Addition Signed-off-by: Job Almekinders --- .../infra/online_stores/contrib/postgres.py | 87 ++++++++++--------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 623637acd5..57353a5068 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -63,55 +63,56 @@ def online_write_batch( Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] ], progress: Optional[Callable[[int], Any]], + batch_size: int = 5000, ) -> None: - project = config.project + # Format insert values + insert_values = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) - with self._get_conn(config) as conn, conn.cursor() as cur: - insert_values = [] - for entity_key, values, timestamp, created_ts in data: - entity_key_bin = serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - timestamp = _to_naive_utc(timestamp) - if created_ts is not None: - created_ts = _to_naive_utc(created_ts) - - for feature_name, val in values.items(): - vector_val = None - if config.online_store.pgvector_enabled: - vector_val = get_list_val_str(val) - insert_values.append( - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - vector_val, - timestamp, - created_ts, - ) + for feature_name, val in values.items(): + vector_val = None + if config.online_store.pgvector_enabled: + vector_val = get_list_val_str(val) + insert_values.append( + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + vector_val, + timestamp, + created_ts, ) - # Control the batch so that we can update the progress - batch_size = 5000 + ) + + # Create insert query + sql_query = sql.SQL( + """ + INSERT INTO {} + (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES %s + ON CONFLICT (entity_key, feature_name) DO + UPDATE SET + value = EXCLUDED.value, + vector_value = EXCLUDED.vector_value, + event_ts = EXCLUDED.event_ts, + created_ts = EXCLUDED.created_ts; + """ + ).format(sql.Identifier(_table_id(config.project, table))) + + # Push data in batches to online store + with self._get_conn(config) as conn, conn.cursor() as cur: for i in range(0, len(insert_values), batch_size): cur_batch = insert_values[i : i + batch_size] - cur.executemany( - sql.SQL( - """ - INSERT INTO {} - (entity_key, feature_name, value, vector_value, event_ts, created_ts) - VALUES %s - ON CONFLICT (entity_key, feature_name) DO - UPDATE SET - value = EXCLUDED.value, - vector_value = EXCLUDED.vector_value, - event_ts = EXCLUDED.event_ts, - created_ts = EXCLUDED.created_ts; - """, - ).format(sql.Identifier(_table_id(project, table))), - cur_batch, - ) + cur.executemany(sql_query, cur_batch) conn.commit() + if progress: progress(len(cur_batch)) From 6f1a4364d9a39c34232e8a7462fe653a50d90fb8 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 16:50:47 +0200 Subject: [PATCH 11/20] Online: Use correct placeholder Signed-off-by: Job Almekinders --- sdk/python/feast/infra/online_stores/contrib/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 57353a5068..428c075143 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -96,7 +96,7 @@ def online_write_batch( """ INSERT INTO {} (entity_key, feature_name, value, vector_value, event_ts, created_ts) - VALUES %s + VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT (entity_key, feature_name) DO UPDATE SET value = EXCLUDED.value, From bfb5e609dfc69255b01a8d50c7361018b7aa6e1b Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 16:59:17 +0200 Subject: [PATCH 12/20] Online: Handle bytes properly in online_read() Signed-off-by: Job Almekinders --- sdk/python/feast/infra/online_stores/contrib/postgres.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 428c075143..dda2c9c8c7 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -170,7 +170,9 @@ def online_read( # when we iterate through the keys since they are in the correct order values_dict = defaultdict(list) for row in rows if rows is not None else []: - values_dict[row[0].tobytes()].append(row[1:]) + values_dict[ + row[0] if isinstance(row[0], bytes) else row[0].tobytes() + ].append(row[1:]) for key in keys: if key in values_dict: From 46f7c964288c553a17c0a4a8b4914a66f8254378 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Thu, 20 Jun 2024 17:27:17 +0200 Subject: [PATCH 13/20] Online: Whitespace Signed-off-by: Job Almekinders --- sdk/python/feast/infra/online_stores/contrib/postgres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index dda2c9c8c7..fbba67cc2d 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -44,6 +44,7 @@ class PostgreSQLOnlineStore(OnlineStore): @contextlib.contextmanager def _get_conn(self, config: RepoConfig): assert config.online_store.type == "postgres" + if config.online_store.conn_type == ConnectionType.pool: if not self._conn_pool: self._conn_pool = _get_connection_pool(config.online_store) From 3d3b88ce35730978f1b1e4434bc1a13ee564d5ca Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 10:48:07 +0200 Subject: [PATCH 14/20] Online: Open ConnectionPool Signed-off-by: Job Almekinders --- sdk/python/feast/infra/online_stores/contrib/postgres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index fbba67cc2d..38be2111fd 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -48,6 +48,7 @@ def _get_conn(self, config: RepoConfig): if config.online_store.conn_type == ConnectionType.pool: if not self._conn_pool: self._conn_pool = _get_connection_pool(config.online_store) + self._conn_pool.open() connection = self._conn_pool.getconn() yield connection self._conn_pool.putconn(connection) From 6625a2c7e033e1026a488dca4273cf97cdc007a0 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 12:40:55 +0200 Subject: [PATCH 15/20] Online: Add typehint Signed-off-by: Job Almekinders --- .../feast/infra/online_stores/contrib/postgres.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 38be2111fd..8715f0f65b 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,7 +2,17 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Literal, + Optional, + Sequence, + Tuple, +) import pytz from psycopg import sql @@ -42,7 +52,7 @@ class PostgreSQLOnlineStore(OnlineStore): _conn_pool: Optional[ConnectionPool] = None @contextlib.contextmanager - def _get_conn(self, config: RepoConfig): + def _get_conn(self, config: RepoConfig) -> Generator[Connection, Any, Any]: assert config.online_store.type == "postgres" if config.online_store.conn_type == ConnectionType.pool: From d0f77ca8d7b8f73a9aa8bda3006376b24983cab9 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Wed, 19 Jun 2024 12:41:38 +0200 Subject: [PATCH 16/20] Utils: Use psycopg3 Use new ConnectionPool Pass kwargs as named argument Use executemany over execute_values Remove not-required open argument in psycopg.connect Improve Use SpooledTemporaryFile Use max_size and add docstring Properly write with StringIO Utils: Use SpooledTemporaryFile over StringIO object Add replace Fix df_to_postgres_table Remove import Utils Signed-off-by: Job Almekinders --- .../infra/utils/postgres/connection_utils.py | 83 ++++++++++--------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/sdk/python/feast/infra/utils/postgres/connection_utils.py b/sdk/python/feast/infra/utils/postgres/connection_utils.py index 3ffc1866bf..e0599019b9 100644 --- a/sdk/python/feast/infra/utils/postgres/connection_utils.py +++ b/sdk/python/feast/infra/utils/postgres/connection_utils.py @@ -1,50 +1,59 @@ -from typing import Dict +from typing import Any, Dict import numpy as np import pandas as pd -import psycopg2 -import psycopg2.extras +import psycopg import pyarrow as pa -from psycopg2.pool import SimpleConnectionPool +from psycopg.connection import Connection +from psycopg_pool import ConnectionPool from feast.infra.utils.postgres.postgres_config import PostgreSQLConfig from feast.type_map import arrow_to_pg_type -def _get_conn(config: PostgreSQLConfig): - conn = psycopg2.connect( - dbname=config.database, - host=config.host, - port=int(config.port), - user=config.user, - password=config.password, - sslmode=config.sslmode, - sslkey=config.sslkey_path, - sslcert=config.sslcert_path, - sslrootcert=config.sslrootcert_path, - options="-c search_path={}".format(config.db_schema or config.user), +def _get_conn(config: PostgreSQLConfig) -> Connection: + """Get a psycopg `Connection`.""" + conn = psycopg.connect( + conninfo=_get_conninfo(config), keepalives_idle=config.keepalives_idle, + **_get_conn_kwargs(config), ) return conn -def _get_connection_pool(config: PostgreSQLConfig): - return SimpleConnectionPool( - config.min_conn, - config.max_conn, - dbname=config.database, - host=config.host, - port=int(config.port), - user=config.user, - password=config.password, - sslmode=config.sslmode, - sslkey=config.sslkey_path, - sslcert=config.sslcert_path, - sslrootcert=config.sslrootcert_path, - options="-c search_path={}".format(config.db_schema or config.user), +def _get_connection_pool(config: PostgreSQLConfig) -> ConnectionPool: + """Get a psycopg `ConnectionPool`.""" + return ConnectionPool( + conninfo=_get_conninfo(config), + min_size=config.min_conn, + max_size=config.max_conn, + open=False, + kwargs=_get_conn_kwargs(config), ) +def _get_conninfo(config: PostgreSQLConfig) -> str: + """Get the `conninfo` argument required for connection objects.""" + return ( + f"postgresql://{config.user}" + f":{config.password}" + f"@{config.host}" + f":{int(config.port)}" + f"/{config.database}" + ) + + +def _get_conn_kwargs(config: PostgreSQLConfig) -> Dict[str, Any]: + """Get the additional `kwargs` required for connection objects.""" + return { + "sslmode": config.sslmode, + "sslkey": config.sslkey_path, + "sslcert": config.sslcert_path, + "sslrootcert": config.sslrootcert_path, + "options": "-c search_path={}".format(config.db_schema or config.user), + } + + def _df_to_create_table_sql(entity_df, table_name) -> str: pa_table = pa.Table.from_pandas(entity_df) columns = [ @@ -63,16 +72,14 @@ def df_to_postgres_table( """ Create a table for the data frame, insert all the values, and return the table schema """ + nr_columns = df.shape[1] + placeholders = ", ".join(["%s"] * nr_columns) + query = f"INSERT INTO {table_name} VALUES ({placeholders})" + values = df.replace({np.NaN: None}).to_numpy().tolist() + with _get_conn(config) as conn, conn.cursor() as cur: cur.execute(_df_to_create_table_sql(df, table_name)) - psycopg2.extras.execute_values( - cur, - f""" - INSERT INTO {table_name} - VALUES %s - """, - df.replace({np.NaN: None}).to_numpy(), - ) + cur.executemany(query, values) return dict(zip(df.columns, df.dtypes)) From b7de4b2c0fb6fd6e2519b3bb8f9ed976bf499c71 Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 13:47:45 +0200 Subject: [PATCH 17/20] Lint: Raise exceptions if cursor returned no columns or rows Add log statement Lint: Fix _to_arrow_internal Lint: Fix _get_entity_df_event_timestamp_range Update exception Use ZeroColumnQueryResult Signed-off-by: Job Almekinders --- sdk/python/feast/errors.py | 10 ++++++++ .../postgres_offline_store/postgres.py | 24 ++++++++++++------- .../postgres_offline_store/postgres_source.py | 9 +++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 22de402f20..3970382dbb 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -401,3 +401,13 @@ def __init__(self, input_dict: dict): super().__init__( f"Failed to serialize the provided dictionary into a pandas DataFrame: {input_dict.keys()}" ) + + +class ZeroRowsQueryResult(Exception): + def __init__(self, query: str): + super().__init__(f"This query returned zero rows:\n{query}") + + +class ZeroColumnQueryResult(Exception): + def __init__(self, query: str): + super().__init__(f"This query returned zero columns:\n{query}") diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index 6a8686d282..caaafb6d29 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -1,4 +1,5 @@ import contextlib +import logging from dataclasses import asdict from datetime import datetime from typing import ( @@ -23,7 +24,7 @@ from pytz import utc from feast.data_source import DataSource -from feast.errors import InvalidEntityType +from feast.errors import InvalidEntityType, ZeroColumnQueryResult, ZeroRowsQueryResult from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL, FeatureView from feast.infra.offline_stores import offline_utils from feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source import ( @@ -276,6 +277,8 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: with _get_conn(self.config.offline_store) as conn, conn.cursor() as cur: conn.read_only = True cur.execute(query) + if not cur.description: + raise ZeroColumnQueryResult(query) fields = [ (c.name, pg_type_code_to_arrow(c.type_code)) for c in cur.description @@ -331,16 +334,19 @@ def _get_entity_df_event_timestamp_range( entity_df_event_timestamp.max().to_pydatetime(), ) elif isinstance(entity_df, str): - # If the entity_df is a string (SQL query), determine range - # from table + # If the entity_df is a string (SQL query), determine range from table with _get_conn(config.offline_store) as conn, conn.cursor() as cur: - ( - cur.execute( - f"SELECT MIN({entity_df_event_timestamp_col}) AS min, MAX({entity_df_event_timestamp_col}) AS max FROM ({entity_df}) as tmp_alias" - ), - ) + query = f""" + SELECT + MIN({entity_df_event_timestamp_col}) AS min, + MAX({entity_df_event_timestamp_col}) AS max + FROM ({entity_df}) AS tmp_alias + """ + cur.execute(query) res = cur.fetchone() - entity_df_event_timestamp_range = (res[0], res[1]) + if not res: + raise ZeroRowsQueryResult(query) + entity_df_event_timestamp_range = (res[0], res[1]) else: raise InvalidEntityType(type(entity_df)) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py index bbb3f768fd..9805c65d50 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py @@ -1,10 +1,11 @@ import json +import logging from typing import Callable, Dict, Iterable, Optional, Tuple from typeguard import typechecked from feast.data_source import DataSource -from feast.errors import DataSourceNoNameException +from feast.errors import DataSourceNoNameException, ZeroColumnQueryResult from feast.infra.utils.postgres.connection_utils import _get_conn from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.protos.feast.core.SavedDataset_pb2 import ( @@ -111,7 +112,11 @@ def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: with _get_conn(config.offline_store) as conn, conn.cursor() as cur: - cur.execute(f"SELECT * FROM {self.get_table_query_string()} AS sub LIMIT 0") + query = f"SELECT * FROM {self.get_table_query_string()} AS sub LIMIT 0" + cur.execute(query) + if not cur.description: + raise ZeroColumnQueryResult(query) + return ( (c.name, pg_type_code_to_pg_type(c.type_code)) for c in cur.description ) From 60f5a23af1caf8cc6bd784d369c77a4c04b69b2a Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 15:12:45 +0200 Subject: [PATCH 18/20] Add comment on +psycopg string Signed-off-by: Job Almekinders --- .../tests/integration/registration/test_universal_registry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 27323e6515..f4abef2a81 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -27,6 +27,7 @@ from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.minio import MinioContainer from testcontainers.mysql import MySqlContainer +from tests.integration.feature_repos.universal.entities import driver from feast import FeatureService, FileSource, RequestSource from feast.data_format import AvroFormat, ParquetFormat @@ -47,7 +48,6 @@ from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType -from tests.integration.feature_repos.universal.entities import driver @pytest.fixture @@ -149,6 +149,8 @@ def pg_registry(): registry_config = RegistryConfig( registry_type="sql", + # The `path` must include `+psycopg` in order for `sqlalchemy.create_engine()` + # to understand that we are using psycopg3. path=f"postgresql+psycopg://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) From f9fa15858c02d80f97e88f41ad58b5af558b448a Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 15:14:37 +0200 Subject: [PATCH 19/20] Docs: Remove mention of psycopg2 Signed-off-by: Job Almekinders --- docs/tutorials/using-scalable-registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md index 30b8e01ed5..25746f60e2 100644 --- a/docs/tutorials/using-scalable-registry.md +++ b/docs/tutorials/using-scalable-registry.md @@ -49,7 +49,7 @@ When this happens, your database is likely using what is referred to as an in `SQLAlchemy` terminology. See your database's documentation for examples on how to set its scheme in the Database URL. -`Psycopg2`, which is the database library leveraged by the online and offline +`Psycopg`, which is the database library leveraged by the online and offline stores, is not impacted by the need to speak a particular dialect, and so the following only applies to the registry. From e2d218870cda17267704e6f5f9e4187ddfe0a4ff Mon Sep 17 00:00:00 2001 From: Job Almekinders Date: Fri, 21 Jun 2024 15:23:45 +0200 Subject: [PATCH 20/20] Lint: Fix --- .../offline_stores/contrib/postgres_offline_store/postgres.py | 1 - .../contrib/postgres_offline_store/postgres_source.py | 1 - .../tests/integration/registration/test_universal_registry.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index caaafb6d29..c4740a960e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -1,5 +1,4 @@ import contextlib -import logging from dataclasses import asdict from datetime import datetime from typing import ( diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py index 9805c65d50..c216328b8d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py @@ -1,5 +1,4 @@ import json -import logging from typing import Callable, Dict, Iterable, Optional, Tuple from typeguard import typechecked diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index f4abef2a81..c119ae800a 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -27,7 +27,6 @@ from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.minio import MinioContainer from testcontainers.mysql import MySqlContainer -from tests.integration.feature_repos.universal.entities import driver from feast import FeatureService, FileSource, RequestSource from feast.data_format import AvroFormat, ParquetFormat @@ -48,6 +47,7 @@ from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType +from tests.integration.feature_repos.universal.entities import driver @pytest.fixture