Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Compare Python objects instead of proto objects #2227

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 175 additions & 27 deletions sdk/python/feast/diff/FcoDiff.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import Generic, Iterable, List, Set, Tuple, TypeVar
from typing import Any, Dict, Generic, Iterable, List, Set, Tuple, TypeVar

from feast.base_feature_view import BaseFeatureView
from feast.diff.property_diff import PropertyDiff, TransitionType
Expand All @@ -16,23 +16,28 @@
from feast.protos.feast.core.RequestFeatureView_pb2 import (
RequestFeatureView as RequestFeatureViewProto,
)
from feast.registry import FeastObjectType, Registry
from feast.repo_contents import RepoContents

FcoProto = TypeVar(
"FcoProto",
EntityProto,
FeatureViewProto,
FeatureServiceProto,
OnDemandFeatureViewProto,
RequestFeatureViewProto,
)
FEAST_OBJECT_TYPE_TO_STR = {
felixwang9817 marked this conversation as resolved.
Show resolved Hide resolved
FeastObjectType.ENTITY: "entity",
FeastObjectType.FEATURE_VIEW: "feature view",
FeastObjectType.ON_DEMAND_FEATURE_VIEW: "on demand feature view",
FeastObjectType.REQUEST_FEATURE_VIEW: "request feature view",
FeastObjectType.FEATURE_SERVICE: "feature service",
}

FEAST_OBJECT_TYPES = FEAST_OBJECT_TYPE_TO_STR.keys()
felixwang9817 marked this conversation as resolved.
Show resolved Hide resolved

Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)


@dataclass
class FcoDiff(Generic[FcoProto]):
class FcoDiff(Generic[Fco]):
name: str
fco_type: str
current_fco: FcoProto
new_fco: FcoProto
current_fco: Fco
new_fco: Fco
fco_property_diffs: List[PropertyDiff]
transition_type: TransitionType

Expand All @@ -48,20 +53,28 @@ def add_fco_diff(self, fco_diff: FcoDiff):
self.fco_diffs.append(fco_diff)


Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)


def tag_objects_for_keep_delete_add(
def tag_objects_for_keep_delete_update_add(
existing_objs: Iterable[Fco], desired_objs: Iterable[Fco]
) -> Tuple[Set[Fco], Set[Fco], Set[Fco]]:
) -> Tuple[Set[Fco], Set[Fco], Set[Fco], Set[Fco]]:
existing_obj_names = {e.name for e in existing_objs}
desired_obj_names = {e.name for e in desired_objs}

objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names}
objs_to_keep = {e for e in desired_objs if e.name in existing_obj_names}
objs_to_update = {e for e in desired_objs if e.name in existing_obj_names}
objs_to_keep = {e for e in existing_objs if e.name in desired_obj_names}
objs_to_delete = {e for e in existing_objs if e.name not in desired_obj_names}

return objs_to_keep, objs_to_delete, objs_to_add
return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add


FcoProto = TypeVar(
"FcoProto",
felixwang9817 marked this conversation as resolved.
Show resolved Hide resolved
EntityProto,
FeatureViewProto,
FeatureServiceProto,
OnDemandFeatureViewProto,
RequestFeatureViewProto,
)


def tag_proto_objects_for_keep_delete_add(
Expand All @@ -80,23 +93,158 @@ def tag_proto_objects_for_keep_delete_add(
FIELDS_TO_IGNORE = {"project"}


def diff_between(current: FcoProto, new: FcoProto, object_type: str) -> FcoDiff:
assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name
def diff_registry_objects(current: Fco, new: Fco, object_type: str) -> FcoDiff:
current_proto = current.to_proto()
new_proto = new.to_proto()
assert current_proto.DESCRIPTOR.full_name == new_proto.DESCRIPTOR.full_name
property_diffs = []
transition: TransitionType = TransitionType.UNCHANGED
if current.spec != new.spec:
for _field in current.spec.DESCRIPTOR.fields:
if current_proto.spec != new_proto.spec:
for _field in current_proto.spec.DESCRIPTOR.fields:
if _field.name in FIELDS_TO_IGNORE:
continue
if getattr(current.spec, _field.name) != getattr(new.spec, _field.name):
if getattr(current_proto.spec, _field.name) != getattr(
new_proto.spec, _field.name
):
transition = TransitionType.UPDATE
property_diffs.append(
PropertyDiff(
_field.name,
getattr(current.spec, _field.name),
getattr(new.spec, _field.name),
getattr(current_proto.spec, _field.name),
getattr(new_proto.spec, _field.name),
)
)
return FcoDiff(
new.spec.name, object_type, current, new, property_diffs, transition,
name=new_proto.spec.name,
fco_type=object_type,
current_fco=current,
new_fco=new,
fco_property_diffs=property_diffs,
transition_type=transition,
)


def extract_objects_for_keep_delete_update_add(
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
) -> Tuple[
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
]:
"""
Returns the objects in the registry that must be modified to achieve the desired repo state.

Args:
registry: The registry storing the current repo state.
current_project: The Feast project whose objects should be compared.
desired_repo_contents: The desired repo state.
"""
objs_to_keep = {}
objs_to_delete = {}
objs_to_update = {}
objs_to_add = {}

registry_object_type_to_objects: Dict[FeastObjectType, List[Any]]
registry_object_type_to_objects = {
felixwang9817 marked this conversation as resolved.
Show resolved Hide resolved
FeastObjectType.ENTITY: registry.list_entities(project=current_project),
FeastObjectType.FEATURE_VIEW: registry.list_feature_views(
project=current_project
),
FeastObjectType.ON_DEMAND_FEATURE_VIEW: registry.list_on_demand_feature_views(
project=current_project
),
FeastObjectType.REQUEST_FEATURE_VIEW: registry.list_request_feature_views(
project=current_project
),
FeastObjectType.FEATURE_SERVICE: registry.list_feature_services(
project=current_project
),
}
registry_object_type_to_repo_contents: Dict[FeastObjectType, Set[Any]]
registry_object_type_to_repo_contents = {
FeastObjectType.ENTITY: desired_repo_contents.entities,
FeastObjectType.FEATURE_VIEW: desired_repo_contents.feature_views,
FeastObjectType.ON_DEMAND_FEATURE_VIEW: desired_repo_contents.on_demand_feature_views,
FeastObjectType.REQUEST_FEATURE_VIEW: desired_repo_contents.request_feature_views,
FeastObjectType.FEATURE_SERVICE: desired_repo_contents.feature_services,
}

for object_type in FEAST_OBJECT_TYPES:
(
to_keep,
to_delete,
to_update,
to_add,
) = tag_objects_for_keep_delete_update_add(
registry_object_type_to_objects[object_type],
registry_object_type_to_repo_contents[object_type],
)

objs_to_keep[object_type] = to_keep
objs_to_delete[object_type] = to_delete
objs_to_update[object_type] = to_update
objs_to_add[object_type] = to_add

return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add


def diff_between(
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
) -> RegistryDiff:
"""
Returns the difference between the current and desired repo states.

Args:
registry: The registry storing the current repo state.
current_project: The Feast project for which the diff is being computed.
desired_repo_contents: The desired repo state.
"""
diff = RegistryDiff()

(
objs_to_keep,
objs_to_delete,
objs_to_update,
objs_to_add,
) = extract_objects_for_keep_delete_update_add(
registry, current_project, desired_repo_contents
)

for object_type in FEAST_OBJECT_TYPES:
objects_to_keep = objs_to_keep[object_type]
objects_to_delete = objs_to_delete[object_type]
objects_to_update = objs_to_update[object_type]
objects_to_add = objs_to_add[object_type]

for e in objects_to_add:
diff.add_fco_diff(
FcoDiff(
name=e.name,
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
current_fco=None,
new_fco=e,
fco_property_diffs=[],
transition_type=TransitionType.CREATE,
)
)
for e in objects_to_delete:
diff.add_fco_diff(
FcoDiff(
name=e.name,
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
current_fco=e,
new_fco=None,
fco_property_diffs=[],
transition_type=TransitionType.DELETE,
)
)
for e in objects_to_update:
current_obj = [_e for _e in objects_to_keep if _e.name == e.name][0]
diff.add_fco_diff(
diff_registry_objects(
current_obj, e, FEAST_OBJECT_TYPE_TO_STR[object_type]
)
)

return diff
76 changes: 63 additions & 13 deletions sdk/python/feast/feature_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ class FeatureService:
Services.
"""

name: str
feature_view_projections: List[FeatureViewProjection]
tags: Dict[str, str]
description: Optional[str] = None
created_timestamp: Optional[datetime] = None
last_updated_timestamp: Optional[datetime] = None
_name: str
_feature_view_projections: List[FeatureViewProjection]
_tags: Dict[str, str]
_description: Optional[str] = None
_created_timestamp: Optional[datetime] = None
_last_updated_timestamp: Optional[datetime] = None

@log_exceptions
def __init__(
Expand All @@ -51,22 +51,22 @@ def __init__(
Raises:
ValueError: If one of the specified features is not a valid type.
"""
self.name = name
self.feature_view_projections = []
self._name = name
self._feature_view_projections = []

for feature_grouping in features:
if isinstance(feature_grouping, BaseFeatureView):
self.feature_view_projections.append(feature_grouping.projection)
self._feature_view_projections.append(feature_grouping.projection)
else:
raise ValueError(
"The FeatureService {fs_name} has been provided with an invalid type"
f'{type(feature_grouping)} as part of the "features" argument.)'
)

self.tags = tags or {}
self.description = description
self.created_timestamp = None
self.last_updated_timestamp = None
self._tags = tags or {}
self._description = description
self._created_timestamp = None
self._last_updated_timestamp = None

def __repr__(self):
items = (f"{k} = {v}" for k, v in self.__dict__.items())
Expand All @@ -93,6 +93,56 @@ def __eq__(self, other):

return True

@property
def name(self) -> str:
return self._name

@name.setter
def name(self, name: str):
self._name = name

@property
def feature_view_projections(self) -> List[FeatureViewProjection]:
return self._feature_view_projections

@feature_view_projections.setter
def feature_view_projections(
self, feature_view_projections: List[FeatureViewProjection]
):
self._feature_view_projections = feature_view_projections

@property
def tags(self) -> Dict[str, str]:
return self._tags

@tags.setter
def tags(self, tags: Dict[str, str]):
self._tags = tags

@property
def description(self) -> Optional[str]:
return self._description

@description.setter
def description(self, description: str):
self._description = description

@property
def created_timestamp(self) -> Optional[datetime]:
return self._created_timestamp

@created_timestamp.setter
def created_timestamp(self, created_timestamp: datetime):
self._created_timestamp = created_timestamp

@property
def last_updated_timestamp(self) -> Optional[datetime]:
return self._last_updated_timestamp

@last_updated_timestamp.setter
def last_updated_timestamp(self, last_updated_timestamp: datetime):
self._last_updated_timestamp = last_updated_timestamp

@staticmethod
def from_proto(feature_service_proto: FeatureServiceProto):
"""
Expand Down
Loading