From 9cd9f748147ce7600413772f24dc4d1943526309 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 15 Oct 2021 17:58:08 -0400 Subject: [PATCH 1/7] Updating roadmap + hero image (#1950) * Update hero image Signed-off-by: Danny Chiao * Fix roadmap extra line Signed-off-by: Danny Chiao Signed-off-by: Amom Mendes --- README.md | 73 ++++++++++++++++++-------------- docs/roadmap.md | 1 - infra/templates/README.md.jinja2 | 2 +- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 1d3d6831de..3a975189af 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Feast is an open source feature store for machine learning. Feast is the fastest Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture - + The above architecture is the minimal Feast deployment. Want to run the full Feast on GCP/AWS? Click [here](https://docs.feast.dev/how-to-guides/feast-gcp-aws). @@ -129,77 +129,86 @@ pprint(feature_vector) The list below contains the functionality that contributors are planning to develop for Feast -* Items below that are in development \(or planned for development\) will be indicated in parentheses. +* Items below that are in development (or planned for development) will be indicated in parentheses. * We welcome contribution to all items in the roadmap! -* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag_yDssO_3nV9qMfxfrkil-wA/viewform). +* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). -* **Data Sources** - * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) - * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) - * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) - * [ ] Kafka source \(Planned for Q4 2021\) - * [ ] HTTP source - * [ ] Snowflake source - * [ ] Synapse source +* **Data Sources** + * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) + * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) + * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) + * [ ] Kafka source (Planned for Q4 2021) + * [ ] Synapse source (Planned for Q4 2021) + * [ ] Snowflake source (Planned for Q4 2021) + * [ ] HTTP source + + * **Offline Stores** * [x] [Redshift](https://docs.feast.dev/reference/offline-stores/redshift) * [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store) - * [x] [Hive \(community maintained\)](https://github.com/baineng/feast-hive) - * [ ] Snowflake - * [ ] Synapse + * [x] [Hive (community maintained)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) + * [ ] Snowflake (Planned for Q4 2021) + * [ ] Synapse (Planned for Q4 2021)\ + * **Online Stores** * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [ ] Postgres + * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) * [ ] Bigtable - * [ ] Cassandra + * [ ] Cassandra\ + * **Streaming** - * [ ] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [ ] Streaming ingestion on AWS \(Planned for Q4 2021\) - * [ ] Streaming ingestion on GCP + * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) + * [ ] Streaming ingestion on AWS (Planned for Q4 2021) + * [ ] Streaming ingestion on GCP\ + * **Feature Engineering** - * [ ] On-demand Transformations \(Development in progress. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)\) - * [ ] Batch transformation \(SQL\) - * [ ] Streaming transformation + * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [ ] Batch transformation (SQL) + * [ ] Streaming transformation\ + * **Deployments** - * [ ] AWS Lambda \(Development in progress. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)\) + * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [ ] Cloud Run * [ ] Kubernetes - * [ ] KNative + * [ ] KNative\ + * **Feature Serving** * [x] Python Client - * [ ] REST Feature Server \(Python\) \(Development in progress. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev_Wr6ZpnLzER4/edit)\) - * [ ] gRPC Feature Server \(Java\) \(See [\#1497](https://github.com/feast-dev/feast/issues/1497)\) + * [x] REST Feature Server (Python) (Alpha release. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev_Wr6ZpnLzER4/edit)) + * [x] gRPC Feature Server (Java) (See [#1497](https://github.com/feast-dev/feast/issues/1497)) * [ ] Java Client * [ ] Go Client * [ ] Push API * [ ] Delete API - * [ ] Feature Logging \(for training\) + * [ ] Feature Logging (for training)\ + * **Data Quality Management** - * [ ] Data profiling and validation \(Great Expectations\) \(Planned for Q4 2021\) + * [ ] Data profiling and validation (Great Expectations) (Planned for Q4 2021) * [ ] Metric production * [ ] Training-serving skew detection * [ ] Drift detection - * [ ] Alerting + * [ ] Alerting\ + * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry - * [x] Model-centric feature tracking \(feature services\) + * [x] Model-centric feature tracking (feature services) * [ ] REST API for browsing feature registry - * [ ] Feast Web UI \(Planned for Q4 2021\) + * [ ] Feast Web UI (Planned for Q4 2021) * [ ] Feature versioning * [ ] Amundsen integration - ## 🎓 Important Resources Please refer to the official documentation at [Documentation](https://docs.feast.dev/) diff --git a/docs/roadmap.md b/docs/roadmap.md index 0f59c04180..66f952eaab 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -10,7 +10,6 @@ The list below contains the functionality that contributors are planning to deve * **Data Sources** - * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index 993fd4cba4..641b1a75f5 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -20,7 +20,7 @@ Feast is an open source feature store for machine learning. Feast is the fastest Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture - + The above architecture is the minimal Feast deployment. Want to run the full Feast on GCP/AWS? Click [here](https://docs.feast.dev/how-to-guides/feast-gcp-aws). From 2a7f2b0322c485766ae233deabc89219b4ab7cd3 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Sun, 17 Oct 2021 18:26:12 -0300 Subject: [PATCH 2/7] :sparkles: Add created timestamp for feature view Signed-off-by: Amom Mendes --- sdk/python/feast/registry.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index ba1397a44e..c3f1a99bfe 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -18,6 +18,7 @@ from urllib.parse import urlparse from google.protobuf.internal.containers import RepeatedCompositeFieldContainer +from google.protobuf.timestamp_pb2 import Timestamp from proto import Message from feast import importer @@ -314,6 +315,10 @@ def apply_feature_view( self._check_conflicting_feature_view_names(feature_view) existing_feature_views_of_same_type: RepeatedCompositeFieldContainer if isinstance(feature_view, FeatureView): + if not feature_view_proto.meta.HasField("created_timestamp"): + feature_view_proto.meta.created_timestamp = Timestamp.FromDatetime( + datetime.now() + ) existing_feature_views_of_same_type = ( self.cached_registry_proto.feature_views ) From 2cd32b91c61db148703f1b773364623b6a7c7620 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Tue, 19 Oct 2021 22:33:56 -0300 Subject: [PATCH 3/7] :art: Adding created timestamp to base feature view and on demand feature view Signed-off-by: Amom Mendes --- sdk/python/feast/base_feature_view.py | 4 +++- sdk/python/feast/feature_view.py | 1 - sdk/python/feast/on_demand_feature_view.py | 4 +++- sdk/python/feast/registry.py | 6 ++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py index 10f949d9a1..97180266d7 100644 --- a/sdk/python/feast/base_feature_view.py +++ b/sdk/python/feast/base_feature_view.py @@ -13,7 +13,8 @@ # limitations under the License. import warnings from abc import ABC, abstractmethod -from typing import List, Type +from datetime import datetime +from typing import List, Optional, Type from google.protobuf.json_format import MessageToJson from proto import Message @@ -32,6 +33,7 @@ def __init__(self, name: str, features: List[Feature]): self._name = name self._features = features self._projection = FeatureViewProjection.from_definition(self) + self.created_timestamp: Optional[datetime] = None @property def name(self) -> str: diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index a8186991cd..e1f2d91c82 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -74,7 +74,6 @@ class FeatureView(BaseFeatureView): input: DataSource batch_source: DataSource stream_source: Optional[DataSource] = None - created_timestamp: Optional[datetime] = None last_updated_timestamp: Optional[datetime] = None materialization_intervals: List[Tuple[datetime, datetime]] diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 95832fa143..aa32ed9720 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,7 +1,8 @@ import copy +from datetime import datetime import functools from types import MethodType -from typing import Dict, List, Type, Union +from typing import Dict, List, Optional, Type, Union import dill import pandas as pd @@ -71,6 +72,7 @@ def __init__( self.input_feature_views[input_ref] = odfv_input self.udf = udf + self.created_timestamp: Optional[datetime] = None @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index c3f1a99bfe..a5076f24ca 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -307,6 +307,8 @@ def apply_feature_view( commit: Whether the change should be persisted immediately """ feature_view.ensure_valid() + if not feature_view.created_timestamp: + feature_view.created_timestamp = datetime.now() feature_view_proto = feature_view.to_proto() feature_view_proto.spec.project = project self._prepare_registry_for_changes() @@ -315,10 +317,6 @@ def apply_feature_view( self._check_conflicting_feature_view_names(feature_view) existing_feature_views_of_same_type: RepeatedCompositeFieldContainer if isinstance(feature_view, FeatureView): - if not feature_view_proto.meta.HasField("created_timestamp"): - feature_view_proto.meta.created_timestamp = Timestamp.FromDatetime( - datetime.now() - ) existing_feature_views_of_same_type = ( self.cached_registry_proto.feature_views ) From 63bae67c25f38a88565900578709b8f22c081ff5 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Tue, 26 Oct 2021 01:35:52 -0300 Subject: [PATCH 4/7] :rotating_light: Fix import Signed-off-by: Amom Mendes --- sdk/python/feast/on_demand_feature_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index aa32ed9720..ea0c761bf9 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,6 +1,6 @@ import copy -from datetime import datetime import functools +from datetime import datetime from types import MethodType from typing import Dict, List, Optional, Type, Union From cf416401ac77704d9c9a18bf239a1c216f990271 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Tue, 26 Oct 2021 01:45:14 -0300 Subject: [PATCH 5/7] :fire: Remove unused import Signed-off-by: Amom Mendes --- sdk/python/feast/registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index a5076f24ca..e5dc8c723e 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -18,7 +18,6 @@ from urllib.parse import urlparse from google.protobuf.internal.containers import RepeatedCompositeFieldContainer -from google.protobuf.timestamp_pb2 import Timestamp from proto import Message from feast import importer From d903906db75739b122f423b86701eb19e9d4f34f Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Sun, 14 Nov 2021 13:59:25 -0300 Subject: [PATCH 6/7] :art: Add read/write created timestamp to ODFV Signed-off-by: Amom Mendes --- README.md | 1 + protos/feast/core/OnDemandFeatureView.proto | 9 +++++++++ sdk/python/feast/on_demand_feature_view.py | 15 +++++++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 42e2553d5e..e4a7ea1d83 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ The list below contains the functionality that contributors are planning to deve * We welcome contribution to all items in the roadmap! * Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag\_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). + * **Data Sources** * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index e116941697..4cfac4cbd0 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -22,6 +22,7 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; option java_outer_classname = "OnDemandFeatureViewProto"; option java_package = "feast.proto.core"; +import "google/protobuf/timestamp.proto"; import "feast/core/FeatureView.proto"; import "feast/core/Feature.proto"; import "feast/core/DataSource.proto"; @@ -29,6 +30,7 @@ import "feast/core/DataSource.proto"; message OnDemandFeatureView { // User-specified specifications of this feature view. OnDemandFeatureViewSpec spec = 1; + OnDemandFeatureViewMeta meta = 2; } message OnDemandFeatureViewSpec { @@ -45,6 +47,13 @@ message OnDemandFeatureViewSpec { map inputs = 4; UserDefinedFunction user_defined_function = 5; + + +} + +message OnDemandFeatureViewMeta { + // Time where this Feature View is created + google.protobuf.Timestamp created_timestamp = 1; } message OnDemandInput { diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 566b80925f..1a38a4f260 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,8 +1,7 @@ import copy import functools -from datetime import datetime from types import MethodType -from typing import Dict, List, Optional, Type, Union +from typing import Dict, List, Type, Union import dill import pandas as pd @@ -19,6 +18,7 @@ ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureViewSpec, + OnDemandFeatureViewMeta, OnDemandInput, ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( @@ -72,7 +72,6 @@ def __init__( self.input_feature_views[input_ref] = odfv_input self.udf = udf - self.created_timestamp: Optional[datetime] = None @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: @@ -92,6 +91,9 @@ def to_proto(self) -> OnDemandFeatureViewProto: Returns: A OnDemandFeatureViewProto protobuf. """ + meta = OnDemandFeatureViewMeta() + if self.created_timestamp: + meta.created_timestamp.FromDatetime(self.created_timestamp) inputs = {} for input_ref, fv in self.input_feature_views.items(): inputs[input_ref] = OnDemandInput(feature_view=fv.to_proto()) @@ -109,7 +111,7 @@ def to_proto(self) -> OnDemandFeatureViewProto: ), ) - return OnDemandFeatureViewProto(spec=spec) + return OnDemandFeatureViewProto(spec=spec, meta=meta) @classmethod def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): @@ -157,6 +159,11 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): on_demand_feature_view_obj ) + if on_demand_feature_view_proto.meta.HasField("created_timestamp"): + on_demand_feature_view_obj.created_timestamp = ( + on_demand_feature_view_proto.meta.created_timestamp.ToDatetime() + ) + return on_demand_feature_view_obj def get_request_data_schema(self) -> Dict[str, ValueType]: From fc4bcab8e987509583cbf1ce470f3f5711c72317 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Mon, 15 Nov 2021 01:03:40 -0300 Subject: [PATCH 7/7] :rotating_light: Fix isort lint Signed-off-by: Amom Mendes --- sdk/python/feast/on_demand_feature_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 1a38a4f260..5e3c7b16e8 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -17,8 +17,8 @@ OnDemandFeatureView as OnDemandFeatureViewProto, ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( - OnDemandFeatureViewSpec, OnDemandFeatureViewMeta, + OnDemandFeatureViewSpec, OnDemandInput, ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import (