From 38de02ffa0d781bb5026219079e2f8e009bcd021 Mon Sep 17 00:00:00 2001 From: davidschuler-8451 Date: Wed, 5 Apr 2023 15:18:04 -0400 Subject: [PATCH 1/3] adding SqlRegistryConfig class Signed-off-by: davidschuler-8451 --- sdk/python/feast/infra/registry/sql.py | 4 ++-- sdk/python/feast/repo_config.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 628b6d1e65..f8f6153291 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -60,7 +60,7 @@ from feast.protos.feast.core.ValidationProfile_pb2 import ( ValidationReference as ValidationReferenceProto, ) -from feast.repo_config import RegistryConfig +from feast.repo_config import SqlRegistryConfig from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView @@ -181,7 +181,7 @@ class FeastMetadataKeys(Enum): class SqlRegistry(BaseRegistry): def __init__( self, - registry_config: Optional[RegistryConfig], + registry_config: Optional[SqlRegistryConfig], project: str, repo_path: Optional[Path], ): diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 3461ae058b..49840a2f45 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -130,6 +130,30 @@ class RegistryConfig(FeastBaseModel): """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ +class SqlRegistryConfig(FeastBaseModel): + """Metadata Store Configuration. Configuration that relates to reading from and writing to the Feast registry.""" + + registry_type: StrictStr = "sql" + """ str: Provider name or a class name that implements Registry.""" + + registry_store_type: Optional[StrictStr] + """ str: Provider name or a class name that implements RegistryStore. """ + + path: StrictStr = "" + """ str: Path to metadata store. + If registry_type is 'file', then an be a local path, or remote object storage path, e.g. a GCS URI + If registry_type is 'sql', then this is a database URL as expected by SQLAlchemy """ + + cache_ttl_seconds: StrictInt = 600 + """int: The cache TTL is the amount of time registry state will be cached in memory. If this TTL is exceeded then + the registry will be refreshed when any feature store method asks for access to registry state. The TTL can be + set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never + expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ + + s3_additional_kwargs: Optional[Dict[str, str]] + """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ + + class RepoConfig(FeastBaseModel): """Repo config. Typically loaded from `feature_store.yaml`""" From 4c1e16f6ff0e2c8579d1c82fa7aa1ea263cf2a46 Mon Sep 17 00:00:00 2001 From: davidschuler-8451 Date: Thu, 6 Apr 2023 08:19:33 -0400 Subject: [PATCH 2/3] refactor: move SqlRegistryConfig class to sql.py Signed-off-by: davidschuler-8451 --- sdk/python/feast/infra/registry/sql.py | 12 +++++++++++- sdk/python/feast/repo_config.py | 24 ------------------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index f8f6153291..522a3e7d2b 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -5,6 +5,7 @@ from threading import Lock from typing import Any, Callable, List, Optional, Set, Union +from pydantic import StrictStr from sqlalchemy import ( # type: ignore BigInteger, Column, @@ -60,7 +61,7 @@ from feast.protos.feast.core.ValidationProfile_pb2 import ( ValidationReference as ValidationReferenceProto, ) -from feast.repo_config import SqlRegistryConfig +from feast.repo_config import RegistryConfig from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView @@ -178,6 +179,15 @@ class FeastMetadataKeys(Enum): ) +class SqlRegistryConfig(RegistryConfig): + registry_type: StrictStr = "sql" + """ str: Provider name or a class name that implements Registry.""" + + path: StrictStr = "" + """ str: Path to metadata store. + If registry_type is 'sql', then this is a database URL as expected by SQLAlchemy """ + + class SqlRegistry(BaseRegistry): def __init__( self, diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 49840a2f45..3461ae058b 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -130,30 +130,6 @@ class RegistryConfig(FeastBaseModel): """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ -class SqlRegistryConfig(FeastBaseModel): - """Metadata Store Configuration. Configuration that relates to reading from and writing to the Feast registry.""" - - registry_type: StrictStr = "sql" - """ str: Provider name or a class name that implements Registry.""" - - registry_store_type: Optional[StrictStr] - """ str: Provider name or a class name that implements RegistryStore. """ - - path: StrictStr = "" - """ str: Path to metadata store. - If registry_type is 'file', then an be a local path, or remote object storage path, e.g. a GCS URI - If registry_type is 'sql', then this is a database URL as expected by SQLAlchemy """ - - cache_ttl_seconds: StrictInt = 600 - """int: The cache TTL is the amount of time registry state will be cached in memory. If this TTL is exceeded then - the registry will be refreshed when any feature store method asks for access to registry state. The TTL can be - set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never - expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ - - s3_additional_kwargs: Optional[Dict[str, str]] - """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ - - class RepoConfig(FeastBaseModel): """Repo config. Typically loaded from `feature_store.yaml`""" From e7864dc727c183a2b187f95b7dc73b2d60960c19 Mon Sep 17 00:00:00 2001 From: davidschuler-8451 Date: Mon, 10 Apr 2023 08:17:48 -0400 Subject: [PATCH 3/3] enabling SqlRegistry to accept RegistryConfig or SqlRegistryConfig Signed-off-by: davidschuler-8451 --- sdk/python/feast/infra/registry/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 522a3e7d2b..14a85e9ad9 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -191,7 +191,7 @@ class SqlRegistryConfig(RegistryConfig): class SqlRegistry(BaseRegistry): def __init__( self, - registry_config: Optional[SqlRegistryConfig], + registry_config: Optional[Union[RegistryConfig, SqlRegistryConfig]], project: str, repo_path: Optional[Path], ):