Update serving service to handle new redis encoding (#1038)

* Update protos Signed-off-by: Terence <terencelimxp@gmail.com> * Add common functions Signed-off-by: Terence <terencelimxp@gmail.com> * Add serving with new encoding and IT Signed-off-by: Terence <terencelimxp@gmail.com> * Update golang protos and mocks Signed-off-by: Terence <terencelimxp@gmail.com> * Remove additional line Signed-off-by: Terence <terencelimxp@gmail.com> * Use single hmget Signed-off-by: Terence <terencelimxp@gmail.com> * Optimize redis hmget call and add maxage test Signed-off-by: Terence <terencelimxp@gmail.com> * Add redis cluster and refactor code duplication Signed-off-by: Terence <terencelimxp@gmail.com> * Attempt to fix IT port in use error Signed-off-by: Terence <terencelimxp@gmail.com> * Some cleanups Signed-off-by: Terence <terencelimxp@gmail.com> * Refactor duplicated online retriever code Signed-off-by: Terence <terencelimxp@gmail.com> * Revert accidental deletion in go.sum Signed-off-by: Terence <terencelimxp@gmail.com> * Fix test utils Signed-off-by: Terence <terencelimxp@gmail.com> * Use immutable map Signed-off-by: Terence <terencelimxp@gmail.com> * Rename redis client adapter and add more tests Signed-off-by: Terence <terencelimxp@gmail.com> * Make properties private Signed-off-by: Terence <terencelimxp@gmail.com> * Cleanup code Signed-off-by: Terence <terencelimxp@gmail.com>
feast-dev · Oct 14, 2020 · 08b37c1 · 08b37c1
1 parent 2a53800
commit 08b37c1
Show file tree

Hide file tree

Showing 39 changed files with 2,847 additions and 298 deletions.
diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java
@@ -17,7 +17,9 @@
 package feast.common.it;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.Duration;
+import com.google.protobuf.Timestamp;
 import feast.proto.core.DataSourceProto.DataSource;
 import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions;
 import feast.proto.core.DataSourceProto.DataSource.FileOptions;
@@ -29,6 +31,7 @@
 import feast.proto.core.FeatureTableProto.FeatureTableSpec;
 import feast.proto.core.SourceProto;
 import feast.proto.core.StoreProto;
+import feast.proto.serving.ServingAPIProto;
 import feast.proto.types.ValueProto;
 import java.util.Collections;
 import java.util.HashMap;
@@ -237,6 +240,31 @@ public static FeatureTableSpec createFeatureTableSpec(
         .build();
   }
 
+  public static FeatureTableSpec createFeatureTableSpec(
+      String name,
+      List<String> entities,
+      ImmutableMap<String, ValueProto.ValueType.Enum> features,
+      int maxAgeSecs,
+      Map<String, String> labels) {
+
+    return FeatureTableSpec.newBuilder()
+        .setName(name)
+        .addAllEntities(entities)
+        .addAllFeatures(
+            features.entrySet().stream()
+                .map(
+                    entry ->
+                        FeatureSpecV2.newBuilder()
+                            .setName(entry.getKey())
+                            .setValueType(entry.getValue())
+                            .putAllLabels(labels)
+                            .build())
+                .collect(Collectors.toList()))
+        .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSecs).build())
+        .putAllLabels(labels)
+        .build();
+  }
+
   public static DataSource createFileDataSourceSpec(
       String fileURL, String fileFormat, String timestampColumn, String datePartitionColumn) {
     return DataSource.newBuilder()
@@ -271,4 +299,32 @@ public static DataSource createKafkaDataSourceSpec(
         .setEventTimestampColumn(timestampColumn)
         .build();
   }
+
+  public static ValueProto.Value createEmptyValue() {
+    return ValueProto.Value.newBuilder().build();
+  }
+
+  public static ValueProto.Value createDoubleValue(double value) {
+    return ValueProto.Value.newBuilder().setDoubleVal(value).build();
+  }
+
+  public static ValueProto.Value createInt64Value(long value) {
+    return ValueProto.Value.newBuilder().setInt64Val(value).build();
+  }
+
+  public static ServingAPIProto.FeatureReferenceV2 createFeatureReference(
+      String featureTableName, String featureName) {
+    return ServingAPIProto.FeatureReferenceV2.newBuilder()
+        .setFeatureTable(featureTableName)
+        .setName(featureName)
+        .build();
+  }
+
+  public static ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow createEntityRow(
+      String entityName, ValueProto.Value entityValue, long seconds) {
+    return ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow.newBuilder()
+        .setTimestamp(Timestamp.newBuilder().setSeconds(seconds))
+        .putFields(entityName, entityValue)
+        .build();
+  }
 }
diff --git a/common/src/main/java/feast/common/models/FeatureTable.java b/common/src/main/java/feast/common/models/FeatureTable.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.common.models;
+
+import feast.proto.core.FeatureTableProto.FeatureTableSpec;
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+
+public class FeatureTable {
+
+  /**
+   * Accepts FeatureTableSpec object and returns its reference in String
+   * "project/featuretable_name".
+   *
+   * @param featureTableSpec {@link FeatureTableSpec}
+   * @return String format of FeatureTableReference
+   */
+  public static String getFeatureTableStringRef(String project, FeatureTableSpec featureTableSpec) {
+    return String.format("%s/%s", project, featureTableSpec.getName());
+  }
+
+  /**
+   * Accepts FeatureReferenceV2 object and returns its reference in String
+   * "project/featuretable_name".
+   *
+   * @param featureReference {@link FeatureReferenceV2}
+   * @return String format of FeatureTableReference
+   */
+  public static String getFeatureTableStringRef(
+      String project, FeatureReferenceV2 featureReference) {
+    return String.format("%s/%s", project, featureReference.getFeatureTable());
+  }
+}
diff --git a/common/src/main/java/feast/common/models/FeatureV2.java b/common/src/main/java/feast/common/models/FeatureV2.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.common.models;
+
+import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+
+public class FeatureV2 {
+
+  /**
+   * Accepts FeatureReferenceV2 object and returns its reference in String
+   * "featuretable_name:feature_name".
+   *
+   * @param featureReference {@link FeatureReferenceV2}
+   * @return String format of FeatureReferenceV2
+   */
+  public static String getFeatureStringRef(FeatureReferenceV2 featureReference) {
+    String ref = featureReference.getName();
+    if (!featureReference.getFeatureTable().isEmpty()) {
+      ref = featureReference.getFeatureTable() + ":" + ref;
+    }
+    return ref;
+  }
+}
diff --git a/go.mod b/go.mod
@@ -25,7 +25,7 @@ require (
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect
-	golang.org/x/tools v0.0.0-20201013053347-2db1cd791039 // indirect
+	golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 // indirect
 	google.golang.org/grpc v1.29.1
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect

diff --git a/go.sum b/go.sum
@@ -490,6 +490,8 @@ golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0
 golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/tools v0.0.0-20201001230009-b5b87423c93b h1:07IVqnnzaip3TGyl/cy32V5YP3FguWG4BybYDTBNpm0=
 golang.org/x/tools v0.0.0-20201001230009-b5b87423c93b/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 h1:BANdcOVw3KTuUiyfDp7wrzCpkCe8UP3lowugJngxBTg=
+golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/tools v0.0.0-20201013053347-2db1cd791039 h1:kLBxO4OPBgPwjg8Vvu+/0DCHIfDwYIGNFcD66NU9kpo=
 golang.org/x/tools v0.0.0-20201013053347-2db1cd791039/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto
@@ -33,6 +33,9 @@ service ServingService {
     // Get online features synchronously.
     rpc GetOnlineFeatures (GetOnlineFeaturesRequest) returns (GetOnlineFeaturesResponse);
 
+    // Get online features (v2) synchronously.
+    rpc GetOnlineFeaturesV2 (GetOnlineFeaturesRequestV2) returns (GetOnlineFeaturesResponse);
+
     // Get batch features asynchronously.
     //
     // The client should check the status of the returned job periodically by
@@ -78,6 +81,14 @@ message FeatureReference {
     reserved 3, 4;
 }
 
+message FeatureReferenceV2 {
+    // Name of the Feature Table to retrieve the feature from.
+    string feature_table = 1;
+
+    // Name of the Feature to retrieve the feature from.
+    string name = 2;
+}
+
 message GetOnlineFeaturesRequest {
     // List of features that are being retrieved
     repeated FeatureReference features = 4;
@@ -106,6 +117,31 @@ message GetOnlineFeaturesRequest {
     }
 }
 
+message GetOnlineFeaturesRequestV2 {
+    // List of features that are being retrieved
+    repeated FeatureReferenceV2 features = 4;
+
+    // List of entity rows, containing entity id and timestamp data.
+    // Used during retrieval of feature rows and for joining feature
+    // rows into a final dataset
+    repeated EntityRow entity_rows = 2;
+
+    // Optional field to specify project name override. If specified, uses the
+    // given project for retrieval. Overrides the projects specified in
+    // Feature References if both are specified.
+    string project = 5;
+
+    message EntityRow {
+      // Request timestamp of this row. This value will be used,
+      // together with maxAge, to determine feature staleness.
+      google.protobuf.Timestamp timestamp = 1;
+
+      // Map containing mapping of entity name to entity value.
+      map<string,feast.types.Value> fields = 2;
+    }
+  }
+
+
 message GetBatchFeaturesRequest {
     // List of features that are being retrieved
     repeated FeatureReference features = 3;

diff --git a/protos/feast/storage/Redis.proto b/protos/feast/storage/Redis.proto
@@ -17,6 +17,7 @@
 syntax = "proto3";
 
 import "feast/types/Field.proto";
+import "feast/types/Value.proto";
 
 package feast.storage;
 
@@ -36,3 +37,11 @@ message RedisKey {
   // by the entity name alphabetically in ascending order.
   repeated feast.types.Field entities = 3;
 }
+
+message RedisKeyV2 {
+  string project = 1;
+
+  repeated string entity_names = 2;
+
+  repeated feast.types.Value entity_values = 3;
+}
diff --git a/sdk/go/mocks/serving_mock.go b/sdk/go/mocks/serving_mock.go