Compare Python objects instead of proto objects (#2227)

* Compare Python objects instead of proto objects

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Remove unnecessary helper method

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix docstring test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add docstring to RepoContents

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Lint

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Update usage test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Set cache ttl to 1 second in tests for local feature server tests

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add FCO test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add properties to feature service

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Lint

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Remove logic that converts Registry to RepoContents

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Always initialize registry

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Move diffing methods from Registry into FcoDiff.py

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix unit test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Put registry initialization back in repo_operations.py

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix usage test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Switch from hardcoded names to enum

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

sdk/python/feast/diff/FcoDiff.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Generic, Iterable, List, Set, Tuple, TypeVar
+from typing import Any, Dict, Generic, Iterable, List, Set, Tuple, TypeVar
 
 from feast.base_feature_view import BaseFeatureView
 from feast.diff.property_diff import PropertyDiff, TransitionType
@@ -16,23 +16,28 @@
 from feast.protos.feast.core.RequestFeatureView_pb2 import (
     RequestFeatureView as RequestFeatureViewProto,
 )
+from feast.registry import FeastObjectType, Registry
+from feast.repo_contents import RepoContents
 
-FcoProto = TypeVar(
-    "FcoProto",
-    EntityProto,
-    FeatureViewProto,
-    FeatureServiceProto,
-    OnDemandFeatureViewProto,
-    RequestFeatureViewProto,
-)
+FEAST_OBJECT_TYPE_TO_STR = {
+    FeastObjectType.ENTITY: "entity",
+    FeastObjectType.FEATURE_VIEW: "feature view",
+    FeastObjectType.ON_DEMAND_FEATURE_VIEW: "on demand feature view",
+    FeastObjectType.REQUEST_FEATURE_VIEW: "request feature view",
+    FeastObjectType.FEATURE_SERVICE: "feature service",
+}
+
+FEAST_OBJECT_TYPES = FEAST_OBJECT_TYPE_TO_STR.keys()
+
+Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)
 
 
 @dataclass
-class FcoDiff(Generic[FcoProto]):
+class FcoDiff(Generic[Fco]):
     name: str
     fco_type: str
-    current_fco: FcoProto
-    new_fco: FcoProto
+    current_fco: Fco
+    new_fco: Fco
     fco_property_diffs: List[PropertyDiff]
     transition_type: TransitionType
 
@@ -48,20 +53,28 @@ def add_fco_diff(self, fco_diff: FcoDiff):
         self.fco_diffs.append(fco_diff)
 
 
-Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)
-
-
-def tag_objects_for_keep_delete_add(
+def tag_objects_for_keep_delete_update_add(
     existing_objs: Iterable[Fco], desired_objs: Iterable[Fco]
-) -> Tuple[Set[Fco], Set[Fco], Set[Fco]]:
+) -> Tuple[Set[Fco], Set[Fco], Set[Fco], Set[Fco]]:
     existing_obj_names = {e.name for e in existing_objs}
     desired_obj_names = {e.name for e in desired_objs}
 
     objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names}
-    objs_to_keep = {e for e in desired_objs if e.name in existing_obj_names}
+    objs_to_update = {e for e in desired_objs if e.name in existing_obj_names}
+    objs_to_keep = {e for e in existing_objs if e.name in desired_obj_names}
     objs_to_delete = {e for e in existing_objs if e.name not in desired_obj_names}
 
-    return objs_to_keep, objs_to_delete, objs_to_add
+    return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add
+
+
+FcoProto = TypeVar(
+    "FcoProto",
+    EntityProto,
+    FeatureViewProto,
+    FeatureServiceProto,
+    OnDemandFeatureViewProto,
+    RequestFeatureViewProto,
+)
 
 
 def tag_proto_objects_for_keep_delete_add(
@@ -80,23 +93,158 @@ def tag_proto_objects_for_keep_delete_add(
 FIELDS_TO_IGNORE = {"project"}
 
 
-def diff_between(current: FcoProto, new: FcoProto, object_type: str) -> FcoDiff:
-    assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name
+def diff_registry_objects(current: Fco, new: Fco, object_type: str) -> FcoDiff:
+    current_proto = current.to_proto()
+    new_proto = new.to_proto()
+    assert current_proto.DESCRIPTOR.full_name == new_proto.DESCRIPTOR.full_name
     property_diffs = []
     transition: TransitionType = TransitionType.UNCHANGED
-    if current.spec != new.spec:
-        for _field in current.spec.DESCRIPTOR.fields:
+    if current_proto.spec != new_proto.spec:
+        for _field in current_proto.spec.DESCRIPTOR.fields:
             if _field.name in FIELDS_TO_IGNORE:
                 continue
-            if getattr(current.spec, _field.name) != getattr(new.spec, _field.name):
+            if getattr(current_proto.spec, _field.name) != getattr(
+                new_proto.spec, _field.name
+            ):
                 transition = TransitionType.UPDATE
                 property_diffs.append(
                     PropertyDiff(
                         _field.name,
-                        getattr(current.spec, _field.name),
-                        getattr(new.spec, _field.name),
+                        getattr(current_proto.spec, _field.name),
+                        getattr(new_proto.spec, _field.name),
                     )
                 )
     return FcoDiff(
-        new.spec.name, object_type, current, new, property_diffs, transition,
+        name=new_proto.spec.name,
+        fco_type=object_type,
+        current_fco=current,
+        new_fco=new,
+        fco_property_diffs=property_diffs,
+        transition_type=transition,
     )
+
+
+def extract_objects_for_keep_delete_update_add(
+    registry: Registry, current_project: str, desired_repo_contents: RepoContents,
+) -> Tuple[
+    Dict[FeastObjectType, Set[Fco]],
+    Dict[FeastObjectType, Set[Fco]],
+    Dict[FeastObjectType, Set[Fco]],
+    Dict[FeastObjectType, Set[Fco]],
+]:
+    """
+    Returns the objects in the registry that must be modified to achieve the desired repo state.
+
+    Args:
+        registry: The registry storing the current repo state.
+        current_project: The Feast project whose objects should be compared.
+        desired_repo_contents: The desired repo state.
+    """
+    objs_to_keep = {}
+    objs_to_delete = {}
+    objs_to_update = {}
+    objs_to_add = {}
+
+    registry_object_type_to_objects: Dict[FeastObjectType, List[Any]]
+    registry_object_type_to_objects = {
+        FeastObjectType.ENTITY: registry.list_entities(project=current_project),
+        FeastObjectType.FEATURE_VIEW: registry.list_feature_views(
+            project=current_project
+        ),
+        FeastObjectType.ON_DEMAND_FEATURE_VIEW: registry.list_on_demand_feature_views(
+            project=current_project
+        ),
+        FeastObjectType.REQUEST_FEATURE_VIEW: registry.list_request_feature_views(
+            project=current_project
+        ),
+        FeastObjectType.FEATURE_SERVICE: registry.list_feature_services(
+            project=current_project
+        ),
+    }
+    registry_object_type_to_repo_contents: Dict[FeastObjectType, Set[Any]]
+    registry_object_type_to_repo_contents = {
+        FeastObjectType.ENTITY: desired_repo_contents.entities,
+        FeastObjectType.FEATURE_VIEW: desired_repo_contents.feature_views,
+        FeastObjectType.ON_DEMAND_FEATURE_VIEW: desired_repo_contents.on_demand_feature_views,
+        FeastObjectType.REQUEST_FEATURE_VIEW: desired_repo_contents.request_feature_views,
+        FeastObjectType.FEATURE_SERVICE: desired_repo_contents.feature_services,
+    }
+
+    for object_type in FEAST_OBJECT_TYPES:
+        (
+            to_keep,
+            to_delete,
+            to_update,
+            to_add,
+        ) = tag_objects_for_keep_delete_update_add(
+            registry_object_type_to_objects[object_type],
+            registry_object_type_to_repo_contents[object_type],
+        )
+
+        objs_to_keep[object_type] = to_keep
+        objs_to_delete[object_type] = to_delete
+        objs_to_update[object_type] = to_update
+        objs_to_add[object_type] = to_add
+
+    return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add
+
+
+def diff_between(
+    registry: Registry, current_project: str, desired_repo_contents: RepoContents,
+) -> RegistryDiff:
+    """
+    Returns the difference between the current and desired repo states.
+
+    Args:
+        registry: The registry storing the current repo state.
+        current_project: The Feast project for which the diff is being computed.
+        desired_repo_contents: The desired repo state.
+    """
+    diff = RegistryDiff()
+
+    (
+        objs_to_keep,
+        objs_to_delete,
+        objs_to_update,
+        objs_to_add,
+    ) = extract_objects_for_keep_delete_update_add(
+        registry, current_project, desired_repo_contents
+    )
+
+    for object_type in FEAST_OBJECT_TYPES:
+        objects_to_keep = objs_to_keep[object_type]
+        objects_to_delete = objs_to_delete[object_type]
+        objects_to_update = objs_to_update[object_type]
+        objects_to_add = objs_to_add[object_type]
+
+        for e in objects_to_add:
+            diff.add_fco_diff(
+                FcoDiff(
+                    name=e.name,
+                    fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
+                    current_fco=None,
+                    new_fco=e,
+                    fco_property_diffs=[],
+                    transition_type=TransitionType.CREATE,
+                )
+            )
+        for e in objects_to_delete:
+            diff.add_fco_diff(
+                FcoDiff(
+                    name=e.name,
+                    fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
+                    current_fco=e,
+                    new_fco=None,
+                    fco_property_diffs=[],
+                    transition_type=TransitionType.DELETE,
+                )
+            )
+        for e in objects_to_update:
+            current_obj = [_e for _e in objects_to_keep if _e.name == e.name][0]
+            diff.add_fco_diff(
+                diff_registry_objects(
+                    current_obj, e, FEAST_OBJECT_TYPE_TO_STR[object_type]
+                )
+            )
+
+    return diff

sdk/python/feast/feature_service.py

@@ -30,12 +30,12 @@ class FeatureService:
             Services.
     """
 
-    name: str
-    feature_view_projections: List[FeatureViewProjection]
-    tags: Dict[str, str]
-    description: Optional[str] = None
-    created_timestamp: Optional[datetime] = None
-    last_updated_timestamp: Optional[datetime] = None
+    _name: str
+    _feature_view_projections: List[FeatureViewProjection]
+    _tags: Dict[str, str]
+    _description: Optional[str] = None
+    _created_timestamp: Optional[datetime] = None
+    _last_updated_timestamp: Optional[datetime] = None
 
     @log_exceptions
     def __init__(
@@ -51,22 +51,22 @@ def __init__(
         Raises:
             ValueError: If one of the specified features is not a valid type.
         """
-        self.name = name
-        self.feature_view_projections = []
+        self._name = name
+        self._feature_view_projections = []
 
         for feature_grouping in features:
             if isinstance(feature_grouping, BaseFeatureView):
-                self.feature_view_projections.append(feature_grouping.projection)
+                self._feature_view_projections.append(feature_grouping.projection)
             else:
                 raise ValueError(
                     "The FeatureService {fs_name} has been provided with an invalid type"
                     f'{type(feature_grouping)} as part of the "features" argument.)'
                 )
 
-        self.tags = tags or {}
-        self.description = description
-        self.created_timestamp = None
-        self.last_updated_timestamp = None
+        self._tags = tags or {}
+        self._description = description
+        self._created_timestamp = None
+        self._last_updated_timestamp = None
 
     def __repr__(self):
         items = (f"{k} = {v}" for k, v in self.__dict__.items())
@@ -93,6 +93,56 @@ def __eq__(self, other):
 
         return True
 
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @name.setter
+    def name(self, name: str):
+        self._name = name
+
+    @property
+    def feature_view_projections(self) -> List[FeatureViewProjection]:
+        return self._feature_view_projections
+
+    @feature_view_projections.setter
+    def feature_view_projections(
+        self, feature_view_projections: List[FeatureViewProjection]
+    ):
+        self._feature_view_projections = feature_view_projections
+
+    @property
+    def tags(self) -> Dict[str, str]:
+        return self._tags
+
+    @tags.setter
+    def tags(self, tags: Dict[str, str]):
+        self._tags = tags
+
+    @property
+    def description(self) -> Optional[str]:
+        return self._description
+
+    @description.setter
+    def description(self, description: str):
+        self._description = description
+
+    @property
+    def created_timestamp(self) -> Optional[datetime]:
+        return self._created_timestamp
+
+    @created_timestamp.setter
+    def created_timestamp(self, created_timestamp: datetime):
+        self._created_timestamp = created_timestamp
+
+    @property
+    def last_updated_timestamp(self) -> Optional[datetime]:
+        return self._last_updated_timestamp
+
+    @last_updated_timestamp.setter
+    def last_updated_timestamp(self, last_updated_timestamp: datetime):
+        self._last_updated_timestamp = last_updated_timestamp
+
     @staticmethod
     def from_proto(feature_service_proto: FeatureServiceProto):
         """