From ec11a7cb8d56d8e2e5cda07e06b4c98dcc9d2ba3 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Thu, 15 Feb 2024 11:04:06 -0500 Subject: [PATCH] feat: Update the Pydantic from v1 to v2 (#3948) --- sdk/python/feast/importer.py | 3 +- .../infra/contrib/spark_kafka_processor.py | 4 +- .../feast/infra/contrib/stream_processor.py | 6 +- .../feature_servers/aws_lambda/config.py | 3 +- .../infra/feature_servers/base_config.py | 2 +- .../feature_servers/gcp_cloudrun/config.py | 3 +- .../feature_servers/local_process/config.py | 2 +- .../infra/materialization/snowflake_engine.py | 6 +- .../feast/infra/offline_stores/bigquery.py | 22 ++- .../contrib/athena_offline_store/athena.py | 2 +- .../athena_offline_store/tests/data_source.py | 4 +- .../contrib/mssql_offline_store/mssql.py | 7 +- .../mssql_offline_store/tests/data_source.py | 4 +- .../postgres_offline_store/postgres.py | 2 +- .../tests/data_source.py | 4 +- .../spark_offline_store/tests/data_source.py | 9 +- .../test_config/manual_tests.py | 2 +- .../trino_offline_store/tests/data_source.py | 6 +- .../contrib/trino_offline_store/trino.py | 12 +- sdk/python/feast/infra/offline_stores/file.py | 3 +- .../feast/infra/offline_stores/redshift.py | 16 +- .../feast/infra/offline_stores/snowflake.py | 8 +- .../infra/offline_stores/snowflake_source.py | 6 +- .../feast/infra/online_stores/bigtable.py | 3 +- .../cassandra_online_store.py | 13 +- .../contrib/hbase_online_store/hbase.py | 3 +- .../contrib/mysql_online_store/mysql.py | 4 +- .../infra/online_stores/contrib/postgres.py | 3 +- .../feast/infra/online_stores/datastore.py | 13 +- .../feast/infra/online_stores/dynamodb.py | 3 +- sdk/python/feast/infra/online_stores/redis.py | 2 +- .../feast/infra/online_stores/snowflake.py | 9 +- .../feast/infra/online_stores/sqlite.py | 3 +- .../feast/infra/passthrough_provider.py | 2 +- .../feast/infra/registry/base_registry.py | 36 ++++ sdk/python/feast/infra/registry/snowflake.py | 9 +- .../infra/utils/snowflake/snowflake_utils.py | 6 +- sdk/python/feast/repo_config.py | 165 ++++++++---------- .../requirements/py3.10-ci-requirements.txt | 17 +- .../requirements/py3.10-requirements.txt | 16 +- .../requirements/py3.8-ci-requirements.txt | 16 +- .../requirements/py3.8-requirements.txt | 12 +- .../requirements/py3.9-ci-requirements.txt | 17 +- .../requirements/py3.9-requirements.txt | 8 +- sdk/python/tests/conftest.py | 5 +- .../feature_repos/repo_configuration.py | 16 +- .../universal/data_source_creator.py | 9 +- .../universal/data_sources/bigquery.py | 3 +- .../universal/data_sources/file.py | 9 +- .../universal/data_sources/redshift.py | 5 +- .../universal/data_sources/snowflake.py | 4 +- .../feature_repos/universal/feature_views.py | 3 +- .../universal/online_store_creator.py | 3 +- sdk/python/tests/unit/cli/test_cli_chdir.py | 9 +- .../offline_stores/test_offline_store.py | 5 +- .../infra/offline_stores/test_redshift.py | 1 + .../infra/scaffolding/test_repo_config.py | 12 +- sdk/python/tests/utils/e2e_test_validation.py | 2 +- setup.py | 4 +- 59 files changed, 333 insertions(+), 253 deletions(-) diff --git a/sdk/python/feast/importer.py b/sdk/python/feast/importer.py index d1d7d62901..938d29fe31 100644 --- a/sdk/python/feast/importer.py +++ b/sdk/python/feast/importer.py @@ -1,5 +1,4 @@ import importlib -from typing import Optional from feast.errors import ( FeastClassImportError, @@ -8,7 +7,7 @@ ) -def import_class(module_name: str, class_name: str, class_type: Optional[str] = None): +def import_class(module_name: str, class_name: str, class_type: str = ""): """ Dynamically loads and returns a class from a module. diff --git a/sdk/python/feast/infra/contrib/spark_kafka_processor.py b/sdk/python/feast/infra/contrib/spark_kafka_processor.py index bac1c28b06..fc4a34f17b 100644 --- a/sdk/python/feast/infra/contrib/spark_kafka_processor.py +++ b/sdk/python/feast/infra/contrib/spark_kafka_processor.py @@ -1,5 +1,5 @@ from types import MethodType -from typing import List, Optional +from typing import List, Optional, no_type_check import pandas as pd from pyspark.sql import DataFrame, SparkSession @@ -76,6 +76,8 @@ def ingest_stream_feature_view( online_store_query = self._write_stream_data(transformed_df, to) return online_store_query + # In the line 64 of __init__(), the "data_source" is assigned a stream_source (and has to be KafkaSource as in line 40). + @no_type_check def _ingest_stream_data(self) -> StreamTable: """Only supports json and avro formats currently.""" if self.format == "json": diff --git a/sdk/python/feast/infra/contrib/stream_processor.py b/sdk/python/feast/infra/contrib/stream_processor.py index df4e144f8c..c4620f4ca1 100644 --- a/sdk/python/feast/infra/contrib/stream_processor.py +++ b/sdk/python/feast/infra/contrib/stream_processor.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from types import MethodType from typing import TYPE_CHECKING, Optional @@ -50,6 +50,7 @@ def __init__( self.sfv = sfv self.data_source = data_source + @abstractmethod def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: """ Ingests data from the stream source attached to the stream feature view; transforms the data @@ -57,12 +58,14 @@ def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: """ raise NotImplementedError + @abstractmethod def _ingest_stream_data(self) -> StreamTable: """ Ingests data into a StreamTable. """ raise NotImplementedError + @abstractmethod def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: """ Applies transformations on top of StreamTable object. Since stream engines use lazy @@ -71,6 +74,7 @@ def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: """ raise NotImplementedError + @abstractmethod def _write_stream_data(self, table: StreamTable, to: PushMode) -> None: """ Launches a job to persist stream data to the online store and/or offline store, depending diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py index 31dd879af6..946831a18f 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/base_config.py b/sdk/python/feast/infra/feature_servers/base_config.py index 756dd79b43..1a348032e1 100644 --- a/sdk/python/feast/infra/feature_servers/base_config.py +++ b/sdk/python/feast/infra/feature_servers/base_config.py @@ -30,5 +30,5 @@ class BaseFeatureServerConfig(FeastConfigBaseModel): enabled: StrictBool = False """Whether the feature server should be launched.""" - feature_logging: Optional[FeatureLoggingConfig] + feature_logging: Optional[FeatureLoggingConfig] = None """ Feature logging configuration """ diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py index 8d0c269cf5..ddcbde7924 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/local_process/config.py b/sdk/python/feast/infra/feature_servers/local_process/config.py index bb2e7bdf73..3d97912e4b 100644 --- a/sdk/python/feast/infra/feature_servers/local_process/config.py +++ b/sdk/python/feast/infra/feature_servers/local_process/config.py @@ -1,4 +1,4 @@ -from pydantic.typing import Literal +from typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 36c42cd390..62b23dfade 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -7,7 +7,7 @@ import click import pandas as pd from colorama import Fore, Style -from pydantic import Field, StrictStr +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from tqdm import tqdm @@ -72,9 +72,7 @@ class SnowflakeMaterializationEngineConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) @dataclass diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 0ee82a908e..68420c0664 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -10,6 +10,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import ConstrainedStr, StrictStr, validator -from pydantic.typing import Literal +from pydantic import StrictStr, field_validator from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed from feast import flags_helper @@ -72,13 +72,6 @@ def get_http_client_info(): return http_client_info.ClientInfo(user_agent=get_user_agent()) -class BigQueryTableCreateDisposition(ConstrainedStr): - """Custom constraint for table_create_disposition. To understand more, see: - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition""" - - values = {"CREATE_NEVER", "CREATE_IF_NEEDED"} - - class BigQueryOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for GCP BigQuery""" @@ -102,10 +95,15 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" - table_create_disposition: Optional[BigQueryTableCreateDisposition] = None - """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED.""" + table_create_disposition: Literal[ + "CREATE_NEVER", "CREATE_IF_NEEDED" + ] = "CREATE_IF_NEEDED" + """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED. + Custom constraint for table_create_disposition. To understand more, see: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition + """ - @validator("billing_project_id") + @field_validator("billing_project_id") def project_id_exists(cls, v, values, **kwargs): if v and not values["project_id"]: raise ValueError( diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py index 85a61106aa..ae510171db 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py @@ -8,6 +8,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -18,7 +19,6 @@ import pyarrow import pyarrow as pa from pydantic import StrictStr -from pydantic.typing import Literal from pytz import utc from feast import OnDemandFeatureView diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index f68e109d6c..6b2238830b 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -48,10 +48,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: table_name = destination_name diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py index 849d5cc797..67bae292c3 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py @@ -3,7 +3,7 @@ import warnings from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union import numpy as np import pandas @@ -11,7 +11,6 @@ import pyarrow as pa import sqlalchemy from pydantic.types import StrictStr -from pydantic.typing import Literal from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker @@ -32,7 +31,7 @@ from feast.infra.provider import RetrievalJob from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView -from feast.repo_config import FeastBaseModel, RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import pa_to_mssql_type from feast.usage import log_exceptions_and_usage @@ -43,7 +42,7 @@ EntitySchema = Dict[str, np.dtype] -class MsSqlServerOfflineStoreConfig(FeastBaseModel): +class MsSqlServerOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for SQL Server""" type: Literal["mssql"] = "mssql" diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py index 2604cf7c18..71ce56bdef 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py @@ -64,10 +64,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, - **kwargs, + timestamp_field: Optional[str] = "ts", ) -> DataSource: # Make sure the field mapping is correct and convert the datetime datasources. if timestamp_field in df: diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index c2e95a8648..9b300d7bf4 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -9,6 +9,7 @@ Iterator, KeysView, List, + Literal, Optional, Tuple, Union, @@ -19,7 +20,6 @@ import pyarrow as pa from jinja2 import BaseLoader, Environment from psycopg2 import sql -from pydantic.typing import Literal from pytz import utc from feast.data_source import DataSource diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py index 224fcea30f..46d5c20e97 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py @@ -82,10 +82,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py index 7b4fda3b5f..b978521885 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py @@ -9,6 +9,7 @@ from pyspark.sql import SparkSession from feast.data_source import DataSource +from feast.feature_logging import LoggingDestination from feast.infra.offline_stores.contrib.spark_offline_store.spark import ( SparkOfflineStoreConfig, ) @@ -68,10 +69,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, - **kwargs, + timestamp_field: Optional[str] = "ts", ) -> DataSource: if timestamp_field in df: df[timestamp_field] = pd.to_datetime(df[timestamp_field], utc=True) @@ -119,3 +120,7 @@ def create_saved_dataset_destination(self) -> SavedDatasetSparkStorage: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" + + def create_logged_features_destination(self) -> LoggingDestination: + # No implementation of LoggingDestination for Spark offline store. + return None # type: ignore diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py index 7d31aa90fb..a31d368ea1 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py @@ -8,6 +8,6 @@ FULL_REPO_CONFIGS = [ IntegrationTestRepoConfig( provider="local", - offline_store_creator=TrinoSourceCreator, + offline_store_creator=TrinoSourceCreator, # type: ignore ), ] diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index a5aa53df7a..fcc0c8d0fa 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -81,10 +81,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) self.client.execute_query( @@ -128,4 +128,6 @@ def create_offline_store_config(self) -> FeastConfigBaseModel: catalog="memory", dataset=self.project_name, connector={"type": "memory"}, + user="test", + auth=None, ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index d4cfdb6632..cdc9435024 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, root_validator +from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, model_validator from trino.auth import ( BasicAuthentication, CertificateAuthentication, @@ -98,14 +98,14 @@ class AuthConfig(FeastConfigBaseModel): type: Literal["kerberos", "basic", "jwt", "oauth2", "certificate"] config: Optional[Dict[StrictStr, Any]] - @root_validator - def config_only_nullable_for_oauth2(cls, values): - auth_type = values["type"] - auth_config = values["config"] + @model_validator(mode="after") + def config_only_nullable_for_oauth2(self): + auth_type = self.type + auth_config = self.config if auth_type != "oauth2" and auth_config is None: raise ValueError(f"config cannot be null for auth type '{auth_type}'") - return values + return self def to_trino_auth(self): auth_type = self.type diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 5e4107545f..0e5064ba78 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime from pathlib import Path -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Literal, Optional, Tuple, Union import dask.dataframe as dd import pandas as pd @@ -10,7 +10,6 @@ import pyarrow.dataset import pyarrow.parquet import pytz -from pydantic.typing import Literal from feast.data_source import DataSource from feast.errors import ( diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 6034bf5ac7..2565a569ad 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -9,6 +9,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pyarrow import pyarrow as pa from dateutil import parser -from pydantic import StrictStr, root_validator -from pydantic.typing import Literal +from pydantic import StrictStr, model_validator from pytz import utc from feast import OnDemandFeatureView, RedshiftSource @@ -72,16 +72,16 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): iam_role: StrictStr """ IAM Role for Redshift, granting it access to S3 """ - @root_validator - def require_cluster_and_user_or_workgroup(cls, values): + @model_validator(mode="after") + def require_cluster_and_user_or_workgroup(self): """ Provisioned Redshift clusters: Require cluster_id and user, ignore workgroup Serverless Redshift: Require workgroup, ignore cluster_id and user """ cluster_id, user, workgroup = ( - values.get("cluster_id"), - values.get("user"), - values.get("workgroup"), + self.cluster_id, + self.user, + self.workgroup, ) if not (cluster_id and user) and not workgroup: raise ValueError( @@ -90,7 +90,7 @@ def require_cluster_and_user_or_workgroup(cls, values): elif cluster_id and workgroup: raise ValueError("cannot specify both cluster_id and workgroup") - return values + return self class RedshiftOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index dd13ffc96c..66e7e78651 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -14,6 +14,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -23,8 +24,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, StrictStr -from pydantic.typing import Literal +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from feast import OnDemandFeatureView @@ -119,9 +119,7 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel): convert_timestamp_columns: Optional[bool] = None """ Convert timestamp columns on export to a Parquet-supported format """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index e29197c68d..9a2c6e09bc 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -1,5 +1,5 @@ import warnings -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, no_type_check from typeguard import typechecked @@ -202,6 +202,7 @@ def get_table_query_string(self) -> str: def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.snowflake_type_to_feast_value_type + @no_type_check def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: @@ -292,7 +293,8 @@ def get_table_column_names_and_types( ) return [ - (column["column_name"], column["snowflake_type"]) for column in metadata + (str(column["column_name"]), str(column["snowflake_type"])) + for column in metadata ] diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py index 30561d0840..3a83d23ced 100644 --- a/sdk/python/feast/infra/online_stores/bigtable.py +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -2,13 +2,12 @@ import logging from concurrent import futures from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Set, Tuple import google from google.cloud import bigtable from google.cloud.bigtable import row_filters from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, utils from feast.feature_view import DUMMY_ENTITY_NAME diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index 34a8cab036..c672e18db0 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -20,7 +20,17 @@ import logging from datetime import datetime -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Literal, + Optional, + Sequence, + Tuple, +) from cassandra.auth import PlainTextAuthProvider from cassandra.cluster import ( @@ -34,7 +44,6 @@ from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy from cassandra.query import PreparedStatement from pydantic import StrictFloat, StrictInt, StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index 1da9de89a8..4b2d8ae39c 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -1,12 +1,11 @@ import calendar import struct from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from happybase import ConnectionPool from happybase.connection import DEFAULT_PROTOCOL, DEFAULT_TRANSPORT from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index c09cb126f0..cf07d5fef1 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pymysql import pytz @@ -23,7 +23,7 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): NOTE: The class *must* end with the `OnlineStoreConfig` suffix. """ - type = "mysql" + type: Literal["mysql"] = "mysql" host: Optional[StrictStr] = None user: Optional[StrictStr] = None diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 49f87ddb0a..308528aaec 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,14 +2,13 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import psycopg2 import pytz from psycopg2 import sql from psycopg2.extras import execute_values from psycopg2.pool import SimpleConnectionPool -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index ed4e7612ba..ae96e16c64 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -17,10 +17,19 @@ from multiprocessing.pool import ThreadPool from queue import Empty, Queue from threading import Lock, Thread -from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Tuple, +) from pydantic import PositiveInt, StrictStr -from pydantic.typing import Literal from feast import Entity, utils from feast.errors import FeastProviderLoginError diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index a1eef16f40..a049189de7 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -14,10 +14,9 @@ import itertools import logging from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal, Union from feast import Entity, FeatureView, utils from feast.infra.infra_object import DYNAMODB_INFRA_OBJECT_CLASS_TYPE, InfraObject diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 9561705aaa..ad84e8db7c 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -21,6 +21,7 @@ Callable, Dict, List, + Literal, Optional, Sequence, Tuple, @@ -30,7 +31,6 @@ import pytz from google.protobuf.timestamp_pb2 import Timestamp from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig, utils from feast.infra.online_stores.helpers import _mmh3, _redis_key, _redis_key_prefix diff --git a/sdk/python/feast/infra/online_stores/snowflake.py b/sdk/python/feast/infra/online_stores/snowflake.py index c1a03a2862..f5600249c9 100644 --- a/sdk/python/feast/infra/online_stores/snowflake.py +++ b/sdk/python/feast/infra/online_stores/snowflake.py @@ -2,11 +2,10 @@ import os from binascii import hexlify from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pandas as pd -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr from feast.entity import Entity from feast.feature_view import FeatureView @@ -57,9 +56,7 @@ class SnowflakeOnlineStoreConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOnlineStore(OnlineStore): diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 6949b2bf24..4a6aa28889 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -16,10 +16,9 @@ import sqlite3 from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from pydantic import StrictStr -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 811abe106c..aca18f4856 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -70,7 +70,7 @@ def batch_engine(self) -> BatchMaterializationEngine: if self._batch_engine: return self._batch_engine else: - engine_config = self.repo_config._batch_engine_config + engine_config = self.repo_config.batch_engine_config config_is_dict = False if isinstance(engine_config, str): engine_config_type = engine_config diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index f89b079478..f23a820d23 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -51,6 +51,7 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_entity(self, name: str, project: str, commit: bool = True): @@ -62,6 +63,7 @@ def delete_entity(self, name: str, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: @@ -77,6 +79,7 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti Returns either the specified entity, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: @@ -90,6 +93,7 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] Returns: List of entities """ + raise NotImplementedError # Data source operations @abstractmethod @@ -104,6 +108,7 @@ def apply_data_source( project: Feast project that this data source belongs to commit: Whether to immediately commit to the registry """ + raise NotImplementedError @abstractmethod def delete_data_source(self, name: str, project: str, commit: bool = True): @@ -115,6 +120,7 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): project: Feast project that this data source belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_data_source( @@ -131,6 +137,7 @@ def get_data_source( Returns: Returns either the specified data source, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_data_sources( @@ -146,6 +153,7 @@ def list_data_sources( Returns: List of data sources """ + raise NotImplementedError # Feature service operations @abstractmethod @@ -159,6 +167,7 @@ def apply_feature_service( feature_service: A feature service that will be registered project: Feast project that this entity belongs to """ + raise NotImplementedError @abstractmethod def delete_feature_service(self, name: str, project: str, commit: bool = True): @@ -170,6 +179,7 @@ def delete_feature_service(self, name: str, project: str, commit: bool = True): project: Feast project that this feature service belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_feature_service( @@ -187,6 +197,7 @@ def get_feature_service( Returns either the specified feature service, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_feature_services( @@ -202,6 +213,7 @@ def list_feature_services( Returns: List of feature services """ + raise NotImplementedError # Feature view operations @abstractmethod @@ -216,6 +228,7 @@ def apply_feature_view( project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_feature_view(self, name: str, project: str, commit: bool = True): @@ -227,6 +240,7 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError # stream feature view operations @abstractmethod @@ -245,6 +259,7 @@ def get_stream_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_stream_feature_views( @@ -260,6 +275,7 @@ def list_stream_feature_views( Returns: List of stream feature views """ + raise NotImplementedError # on demand feature view operations @abstractmethod @@ -278,6 +294,7 @@ def get_on_demand_feature_view( Returns either the specified on demand feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_on_demand_feature_views( @@ -293,6 +310,7 @@ def list_on_demand_feature_views( Returns: List of on demand feature views """ + raise NotImplementedError # regular feature view operations @abstractmethod @@ -311,6 +329,7 @@ def get_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_feature_views( @@ -326,6 +345,7 @@ def list_feature_views( Returns: List of feature views """ + raise NotImplementedError # request feature view operations @abstractmethod @@ -344,6 +364,7 @@ def get_request_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_request_feature_views( @@ -359,6 +380,7 @@ def list_request_feature_views( Returns: List of request feature views """ + raise NotImplementedError @abstractmethod def apply_materialization( @@ -379,6 +401,7 @@ def apply_materialization( end_date (datetime): End date of the materialization interval to track commit: Whether the change should be persisted immediately """ + raise NotImplementedError # Saved dataset operations @abstractmethod @@ -396,6 +419,7 @@ def apply_saved_dataset( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_saved_dataset( @@ -413,6 +437,7 @@ def get_saved_dataset( Returns either the specified SavedDataset, or raises an exception if none is found """ + raise NotImplementedError def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): """ @@ -427,6 +452,7 @@ def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = Fals Returns either the specified SavedDataset, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_saved_datasets( @@ -442,6 +468,7 @@ def list_saved_datasets( Returns: Returns the list of SavedDatasets """ + raise NotImplementedError # Validation reference operations @abstractmethod @@ -459,6 +486,7 @@ def apply_validation_reference( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_validation_reference(self, name: str, project: str, commit: bool = True): @@ -470,6 +498,7 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr project: Feast project that this object belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_validation_reference( @@ -487,6 +516,7 @@ def get_validation_reference( Returns either the specified ValidationReference, or raises an exception if none is found """ + raise NotImplementedError # TODO: Needs to be implemented. def list_validation_references( @@ -519,6 +549,7 @@ def list_project_metadata( Returns: List of project metadata """ + raise NotImplementedError @abstractmethod def update_infra(self, infra: Infra, project: str, commit: bool = True): @@ -530,6 +561,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True): project: Feast project that the Infra object refers to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_infra(self, project: str, allow_cache: bool = False) -> Infra: @@ -543,6 +575,7 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra: Returns: The stored Infra object. """ + raise NotImplementedError @abstractmethod def apply_user_metadata( @@ -567,14 +600,17 @@ def proto(self) -> RegistryProto: Returns: The registry proto object. """ + raise NotImplementedError @abstractmethod def commit(self): """Commits the state of the registry cache to the remote registry store.""" + raise NotImplementedError @abstractmethod def refresh(self, project: Optional[str] = None): """Refreshes the state of the registry cache by fetching the registry state from the remote registry store.""" + raise NotImplementedError @staticmethod def _message_to_sorted_dict(message: Message) -> Dict[str, Any]: diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index c1ebf13d6b..cdf79c78b5 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -5,10 +5,9 @@ from datetime import datetime, timedelta from enum import Enum from threading import Lock -from typing import Any, Callable, List, Optional, Set, Union +from typing import Any, Callable, List, Literal, Optional, Set, Union -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr import feast from feast import usage @@ -103,9 +102,7 @@ class SnowflakeRegistryConfig(RegistryConfig): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeRegistry(BaseRegistry): diff --git a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py index 8eb5177ac2..8548e4dbd8 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py +++ b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py @@ -43,11 +43,7 @@ class GetSnowflakeConnection: - def __init__( - self, - config: str, - autocommit=True, - ): + def __init__(self, config: Any, autocommit=True): self.config = config self.autocommit = autocommit diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 3461ae058b..c69bb4d1e7 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -2,20 +2,19 @@ import os import warnings from pathlib import Path -from typing import Any +from typing import Any, Dict, Optional import yaml from pydantic import ( BaseModel, + ConfigDict, Field, StrictInt, StrictStr, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) -from pydantic.error_wrappers import ErrorWrapper -from pydantic.typing import Dict, Optional from feast.errors import ( FeastFeatureServerTypeInvalidError, @@ -93,17 +92,13 @@ class FeastBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "allow" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") class FeastConfigBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "forbid" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") class RegistryConfig(FeastBaseModel): @@ -112,7 +107,7 @@ class RegistryConfig(FeastBaseModel): registry_type: StrictStr = "file" """ str: Provider name or a class name that implements Registry.""" - registry_store_type: Optional[StrictStr] + registry_store_type: Optional[StrictStr] = None """ str: Provider name or a class name that implements RegistryStore. """ path: StrictStr = "" @@ -126,7 +121,7 @@ class RegistryConfig(FeastBaseModel): set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ - s3_additional_kwargs: Optional[Dict[str, str]] + s3_additional_kwargs: Optional[Dict[str, str]] = None """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ @@ -142,7 +137,7 @@ class RepoConfig(FeastBaseModel): provider: StrictStr """ str: local or gcp or aws """ - _registry_config: Any = Field(alias="registry", default="data/registry.db") + registry_config: Any = Field(alias="registry", default="data/registry.db") """ Configures the registry. Can be: 1. str: a path to a file based registry (a local path, or remote object storage path, e.g. a GCS URI) @@ -150,19 +145,19 @@ class RepoConfig(FeastBaseModel): 3. SnowflakeRegistryConfig: Using a Snowflake table to store the registry """ - _online_config: Any = Field(alias="online_store") + online_config: Any = Field(None, alias="online_store") """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ - _offline_config: Any = Field(alias="offline_store") + offline_config: Any = Field(None, alias="offline_store") """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ - _batch_engine_config: Any = Field(alias="batch_engine") + batch_engine_config: Any = Field(None, alias="batch_engine") """ BatchMaterializationEngine: Batch materialization configuration (optional depending on provider)""" - feature_server: Optional[Any] + feature_server: Optional[Any] = None """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ - flags: Any + flags: Any = None """ Flags (deprecated field): Feature flags for experimental features """ repo_path: Optional[Path] = None @@ -187,42 +182,42 @@ def __init__(self, **data: Any): self._registry = None if "registry" not in data: raise FeastRegistryNotSetError() - self._registry_config = data["registry"] + self.registry_config = data["registry"] self._offline_store = None if "offline_store" in data: - self._offline_config = data["offline_store"] + self.offline_config = data["offline_store"] else: if data["provider"] == "local": - self._offline_config = "file" + self.offline_config = "file" elif data["provider"] == "gcp": - self._offline_config = "bigquery" + self.offline_config = "bigquery" elif data["provider"] == "aws": - self._offline_config = "redshift" + self.offline_config = "redshift" elif data["provider"] == "azure": - self._offline_config = "mssql" + self.offline_config = "mssql" self._online_store = None if "online_store" in data: - self._online_config = data["online_store"] + self.online_config = data["online_store"] else: if data["provider"] == "local": - self._online_config = "sqlite" + self.online_config = "sqlite" elif data["provider"] == "gcp": - self._online_config = "datastore" + self.online_config = "datastore" elif data["provider"] == "aws": - self._online_config = "dynamodb" + self.online_config = "dynamodb" elif data["provider"] == "rockset": - self._online_config = "rockset" + self.online_config = "rockset" self._batch_engine = None if "batch_engine" in data: - self._batch_engine_config = data["batch_engine"] + self.batch_engine_config = data["batch_engine"] elif "batch_engine_config" in data: - self._batch_engine_config = data["batch_engine_config"] + self.batch_engine_config = data["batch_engine_config"] else: # Defaults to using local in-process materialization engine. - self._batch_engine_config = "local" + self.batch_engine_config = "local" if isinstance(self.feature_server, Dict): self.feature_server = get_feature_server_config_from_type( @@ -242,71 +237,71 @@ def __init__(self, **data: Any): @property def registry(self): if not self._registry: - if isinstance(self._registry_config, Dict): - if "registry_type" in self._registry_config: + if isinstance(self.registry_config, Dict): + if "registry_type" in self.registry_config: self._registry = get_registry_config_from_type( - self._registry_config["registry_type"] - )(**self._registry_config) + self.registry_config["registry_type"] + )(**self.registry_config) else: # This may be a custom registry store, which does not need a 'registry_type' - self._registry = RegistryConfig(**self._registry_config) - elif isinstance(self._registry_config, str): + self._registry = RegistryConfig(**self.registry_config) + elif isinstance(self.registry_config, str): # User passed in just a path to file registry self._registry = get_registry_config_from_type("file")( - path=self._registry_config + path=self.registry_config ) - elif self._registry_config: - self._registry = self._registry_config + elif self.registry_config: + self._registry = self.registry_config return self._registry @property def offline_store(self): if not self._offline_store: - if isinstance(self._offline_config, Dict): + if isinstance(self.offline_config, Dict): self._offline_store = get_offline_config_from_type( - self._offline_config["type"] - )(**self._offline_config) - elif isinstance(self._offline_config, str): + self.offline_config["type"] + )(**self.offline_config) + elif isinstance(self.offline_config, str): self._offline_store = get_offline_config_from_type( - self._offline_config + self.offline_config )() - elif self._offline_config: - self._offline_store = self._offline_config + elif self.offline_config: + self._offline_store = self.offline_config return self._offline_store @property def online_store(self): if not self._online_store: - if isinstance(self._online_config, Dict): + if isinstance(self.online_config, Dict): self._online_store = get_online_config_from_type( - self._online_config["type"] - )(**self._online_config) - elif isinstance(self._online_config, str): - self._online_store = get_online_config_from_type(self._online_config)() - elif self._online_config: - self._online_store = self._online_config + self.online_config["type"] + )(**self.online_config) + elif isinstance(self.online_config, str): + self._online_store = get_online_config_from_type(self.online_config)() + elif self.online_config: + self._online_store = self.online_config return self._online_store @property def batch_engine(self): if not self._batch_engine: - if isinstance(self._batch_engine_config, Dict): + if isinstance(self.batch_engine_config, Dict): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config["type"] - )(**self._batch_engine_config) - elif isinstance(self._batch_engine_config, str): + self.batch_engine_config["type"] + )(**self.batch_engine_config) + elif isinstance(self.batch_engine_config, str): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config + self.batch_engine_config )() - elif self._batch_engine_config: + elif self.batch_engine_config: self._batch_engine = self._batch_engine return self._batch_engine - @root_validator(pre=True) + @model_validator(mode="before") @log_exceptions - def _validate_online_store_config(cls, values): + def _validate_online_store_config(cls, values: Any) -> Any: # This method will validate whether the online store configurations are set correctly. This explicit validation # is necessary because Pydantic Unions throw very verbose and cryptic exceptions. We also use this method to # impute the default online store type based on the selected provider. For the time being this method should be @@ -347,14 +342,12 @@ def _validate_online_store_config(cls, values): online_config_class = get_online_config_from_type(online_store_type) online_config_class(**values["online_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="online_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_offline_store_config(cls, values): + @model_validator(mode="before") + @classmethod + def _validate_offline_store_config(cls, values: Any) -> Any: # Set empty offline_store config if it isn't set explicitly if "offline_store" not in values: values["offline_store"] = dict() @@ -385,15 +378,13 @@ def _validate_offline_store_config(cls, values): offline_config_class = get_offline_config_from_type(offline_store_type) offline_config_class(**values["offline_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="offline_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_feature_server_config(cls, values): + @model_validator(mode="before") + @classmethod + def _validate_feature_server_config(cls, values: Any) -> Any: # Having no feature server is the default. if "feature_server" not in values: return values @@ -420,15 +411,13 @@ def _validate_feature_server_config(cls, values): ) feature_server_config_class(**values["feature_server"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="feature_server")], - model=RepoConfig, - ) + raise e return values - @validator("project") - def _validate_project_name(cls, v): + @field_validator("project") + @classmethod + def _validate_project_name(cls, v: str) -> str: from feast.repo_operations import is_valid_name if not is_valid_name(v): @@ -438,10 +427,11 @@ def _validate_project_name(cls, v): ) return v - @validator("flags") - def _validate_flags(cls, v): - if not isinstance(v, Dict): - return + @field_validator("flags") + @classmethod + def _validate_flags(cls, v: Optional[dict]) -> Optional[dict]: + if not isinstance(v, dict): + return v _logger.warning( "Flags are no longer necessary in Feast. Experimental features will log warnings instead." @@ -463,8 +453,7 @@ def write_to_path(self, repo_path: Path): sort_keys=False, ) - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class FeastConfigError(Exception): diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index ffb4662eb1..34d0b0c284 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,11 +4,12 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # - alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -225,6 +226,10 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -259,7 +264,7 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -326,7 +331,6 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) - # great-expectations importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 @@ -630,11 +634,13 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 pygments==2.17.2 @@ -927,6 +933,7 @@ typing-extensions==4.9.0 # great-expectations # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # uvicorn diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index d38a287d72..ba474f6120 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,7 +4,8 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # - +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -140,10 +141,12 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) +pydantic-core==2.16.2 + # via pydantic pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 @@ -176,7 +179,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -205,12 +210,15 @@ typing-extensions==4.9.0 # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # uvicorn urllib3==2.2.0 # via requests uvicorn[standard]==0.27.1 - # via feast (setup.py) + # via + # feast (setup.py) + # uvicorn uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 33dd89c362..bf8f4fbc42 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -7,8 +7,10 @@ alabaster==0.7.13 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -230,6 +232,10 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -264,7 +270,7 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -332,7 +338,6 @@ importlib-metadata==6.11.0 # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab @@ -650,11 +655,13 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 pygments==2.17.2 @@ -952,6 +959,7 @@ typing-extensions==4.9.0 # ipython # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # starlette diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 388bb3143f..5e8481e770 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,7 +4,8 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # - +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -145,10 +146,12 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) +pydantic-core==2.16.2 + # via pydantic pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 @@ -210,13 +213,16 @@ typing-extensions==4.9.0 # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette # uvicorn urllib3==2.2.0 # via requests uvicorn[standard]==0.27.1 - # via feast (setup.py) + # via + # feast (setup.py) + # uvicorn uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 9cb322d2f6..670ba1c07d 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,11 +4,12 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # - alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -225,6 +226,10 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -259,7 +264,7 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.8 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -327,7 +332,6 @@ importlib-metadata==6.11.0 # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab @@ -637,11 +641,13 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.2 + # via pydantic pyflakes==3.0.1 # via flake8 pygments==2.17.2 @@ -938,6 +944,7 @@ typing-extensions==4.9.0 # ipython # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # starlette diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 012dac6f81..2815ed0d78 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,7 +4,8 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # - +annotated-types==0.6.0 + # via pydantic anyio==4.2.0 # via # httpx @@ -140,10 +141,12 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.14 +pydantic==2.6.1 # via # fastapi # feast (setup.py) +pydantic-core==2.16.2 + # via pydantic pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 @@ -205,6 +208,7 @@ typing-extensions==4.9.0 # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette # uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 728bd9b34f..743a1ce4a0 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -18,7 +18,7 @@ from datetime import datetime, timedelta from multiprocessing import Process from sys import platform -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, no_type_check import pandas as pd import pytest @@ -187,9 +187,10 @@ def environment(request, worker_id): e.online_store_creator.teardown() -_config_cache = {} +_config_cache: Any = {} +@no_type_check def pytest_generate_tests(metafunc: pytest.Metafunc): """ This function receives each test function (wrapped in Metafunc) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 027dea2c58..f745bafa13 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -99,7 +99,7 @@ "host": os.getenv("ROCKSET_APISERVER", "api.rs2.usw2.rockset.com"), } -OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, DataSourceCreator] = { +OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, Tuple[str, Type[DataSourceCreator]]] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), "redshift": ("aws", RedshiftDataSourceCreator), @@ -111,7 +111,7 @@ ] AVAILABLE_ONLINE_STORES: Dict[ - str, Tuple[Union[str, Dict[str, str]], Optional[Type[OnlineStoreCreator]]] + str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] ] = { "sqlite": ({"type": "sqlite"}, None), } @@ -169,7 +169,7 @@ AVAILABLE_ONLINE_STORES = { c.online_store["type"] if isinstance(c.online_store, dict) - else c.online_store: (c.online_store, c.online_store_creator) + else c.online_store: (c.online_store, c.online_store_creator) # type: ignore for c in FULL_REPO_CONFIGS } @@ -328,7 +328,7 @@ class UniversalFeatureViews: customer: FeatureView global_fv: FeatureView driver: FeatureView - driver_odfv: OnDemandFeatureView + driver_odfv: Optional[OnDemandFeatureView] order: FeatureView location: FeatureView field_mapping: FeatureView @@ -410,9 +410,7 @@ def construct_test_environment( online_creator = test_repo_config.online_store_creator( project, fixture_request=fixture_request ) - online_store = ( - test_repo_config.online_store - ) = online_creator.create_online_store() + online_store = online_creator.create_online_store() else: online_creator = None online_store = test_repo_config.online_store @@ -422,7 +420,7 @@ def construct_test_environment( AwsLambdaFeatureServerConfig, ) - feature_server = AwsLambdaFeatureServerConfig( + feature_server: Any = AwsLambdaFeatureServerConfig( enabled=True, execution_role_name=os.getenv( "AWS_LAMBDA_ROLE", @@ -465,7 +463,7 @@ def construct_test_environment( # Create feature_store.yaml out of the config with open(Path(repo_dir_name) / "feature_store.yaml", "w") as f: - yaml.safe_dump(json.loads(config.json()), f) + yaml.safe_dump(json.loads(config.model_dump_json(by_alias=True)), f) fs = FeatureStore(repo_dir_name) # We need to initialize the registry, because if nothing is applied in the test before tearing down diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index d64463606f..5e5062291d 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -42,19 +42,20 @@ def create_data_source( A Data source object, pointing to a table or file that is uploaded/persisted for the purpose of the test. """ - ... + raise NotImplementedError @abstractmethod def create_offline_store_config(self) -> FeastConfigBaseModel: - ... + raise NotImplementedError @abstractmethod def create_saved_dataset_destination(self) -> SavedDatasetStorage: - ... + raise NotImplementedError + @abstractmethod def create_logged_features_destination(self) -> LoggingDestination: raise NotImplementedError @abstractmethod def teardown(self): - ... + raise NotImplementedError diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 215d19ba7f..066497a0bc 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -64,10 +64,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, - **kwargs, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 3263785683..008bb8d881 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -39,9 +39,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -94,9 +94,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -167,11 +167,10 @@ def _upload_parquet_file(self, df, file_name, minio_endpoint): def create_data_source( self, df: pd.DataFrame, - destination_name: Optional[str] = None, - suffix: Optional[str] = None, - timestamp_field="ts", + destination_name: str, created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: filename = f"{destination_name}.parquet" port = self.minio.get_exposed_port("9000") diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index e6f20d6125..5a4e3f1085 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -42,16 +42,17 @@ def __init__(self, project_name: str, *args, **kwargs): iam_role=os.getenv( "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role" ), + workgroup="", ) def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py index 1414291a18..1481b11a10 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py @@ -48,10 +48,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 5938a0c936..9bb8aae77f 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -14,6 +14,7 @@ StreamFeatureView, ) from feast.data_source import DataSource, RequestSource +from feast.feature_view_projection import FeatureViewProjection from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, @@ -55,7 +56,7 @@ def conv_rate_plus_100(features_df: pd.DataFrame) -> pd.DataFrame: def conv_rate_plus_100_feature_view( - sources: Dict[str, Union[RequestSource, FeatureView]], + sources: List[Union[FeatureView, RequestSource, FeatureViewProjection]], infer_features: bool = False, features: Optional[List[Field]] = None, ) -> OnDemandFeatureView: diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py index 10a8143739..4932001e76 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from feast.repo_config import FeastConfigBaseModel @@ -10,5 +10,6 @@ def __init__(self, project_name: str, **kwargs): def create_online_store(self) -> FeastConfigBaseModel: raise NotImplementedError + @abstractmethod def teardown(self): raise NotImplementedError diff --git a/sdk/python/tests/unit/cli/test_cli_chdir.py b/sdk/python/tests/unit/cli/test_cli_chdir.py index cf1d031227..12ca8f6b08 100644 --- a/sdk/python/tests/unit/cli/test_cli_chdir.py +++ b/sdk/python/tests/unit/cli/test_cli_chdir.py @@ -15,7 +15,7 @@ def test_cli_chdir() -> None: # Make sure the path is absolute by resolving any symlinks temp_path = Path(temp_dir).resolve() result = runner.run(["init", "my_project"], cwd=temp_path) - repo_path = temp_path / "my_project" / "feature_repo" + repo_path = str(temp_path / "my_project" / "feature_repo") assert result.returncode == 0 result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_path) @@ -44,7 +44,12 @@ def test_cli_chdir() -> None: assert result.returncode == 0 result = runner.run( - ["--chdir", repo_path, "materialize-incremental", end_date.isoformat()], + [ + "--chdir", + repo_path, + "materialize-incremental", + end_date.isoformat(), + ], cwd=temp_path, ) assert result.returncode == 0 diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index 220bdba0da..f93237fce5 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -61,12 +61,12 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: return pyarrow.Table() @property - def full_feature_names(self) -> bool: + def full_feature_names(self) -> bool: # type: ignore """Returns True if full feature names should be applied to the results of the query.""" return False @property - def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: # type: ignore """Returns a list containing all the on demand feature views to be handled.""" return [] @@ -118,6 +118,7 @@ def retrieval_job(request, environment): database="feast", s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion", iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role", + workgroup="", ) environment.test_repo_config.offline_store = offline_store_config return RedshiftRetrievalJob( diff --git a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py index 049977489b..48ee99e89f 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py @@ -31,6 +31,7 @@ def test_offline_write_batch( user="user", iam_role="abcdef", s3_staging_location="s3://bucket/path", + workgroup="", ), ) diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index 42229f8683..ca4ed6472b 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -45,8 +45,7 @@ def test_nullable_online_store_aws(): entity_key_serialization_version: 2 """ ), - expect_error="__root__ -> offline_store -> __root__\n" - " please specify either cluster_id & user if using provisioned clusters, or workgroup if using serverless (type=value_error)", + expect_error="4 validation errors for RepoConfig\nregion\n Field required", ) @@ -154,8 +153,7 @@ def test_extra_field(): path: "online_store.db" """ ), - expect_error="__root__ -> online_store -> that_field_should_not_be_here\n" - " extra fields not permitted (type=value_error.extra)", + expect_error="1 validation error for RepoConfig\nthat_field_should_not_be_here\n Extra inputs are not permitted", ) @@ -186,7 +184,7 @@ def test_bad_type(): path: 100500 """ ), - expect_error="__root__ -> online_store -> path\n str type expected", + expect_error="1 validation error for RepoConfig\npath\n Input should be a valid string", ) @@ -201,9 +199,7 @@ def test_no_project(): entity_key_serialization_version: 2 """ ), - expect_error="1 validation error for RepoConfig\n" - "project\n" - " field required (type=value_error.missing)", + expect_error="1 validation error for RepoConfig\nproject\n Field required", ) diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index bacc8c1720..d8c769f12c 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -193,7 +193,7 @@ def make_feature_store_yaml( repo_path=str(Path(repo_dir_name)), entity_key_serialization_version=2, ) - config_dict = config.dict() + config_dict = config.model_dump(by_alias=True) if ( isinstance(config_dict["online_store"], dict) and "redis_type" in config_dict["online_store"] diff --git a/setup.py b/setup.py index ebc4df31a8..a73ef31b06 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ "protobuf<4.23.4,>3.20", "proto-plus>=1.20.0,<2", "pyarrow>=4", - "pydantic>=1,<2", + "pydantic>=2.0.0", "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", @@ -126,7 +126,7 @@ "cassandra-driver>=3.24.0,<4", ] -GE_REQUIRED = ["great_expectations>=0.15.41,<0.16.0"] +GE_REQUIRED = ["great_expectations>=0.15.41"] AZURE_REQUIRED = [ "azure-storage-blob>=0.37.0",