Skip to content

Commit

Permalink
Refactor tests which utilizes feature sets (#1186)
Browse files Browse the repository at this point in the history 
* Update common module

Signed-off-by: Terence <terencelimxp@gmail.com>

* Update core module

Signed-off-by: Terence <terencelimxp@gmail.com>

* Update serving module

Signed-off-by: Terence <terencelimxp@gmail.com>

* Update storage module

Signed-off-by: Terence <terencelimxp@gmail.com>
  • Loading branch information
@terryyylim
terryyylim authored Nov 23, 2020
1 parent aed366b commit 280d87f
Show file tree
Hide file tree
Showing 32 changed files with 591 additions and 3,004 deletions.
89 changes: 4 additions & 85 deletions common-test/src/main/java/feast/common/it/DataGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,11 @@
import feast.proto.core.EntityProto;
import feast.proto.core.FeatureProto;
import feast.proto.core.FeatureProto.FeatureSpecV2;
import feast.proto.core.FeatureSetProto;
import feast.proto.core.FeatureTableProto.FeatureTableSpec;
import feast.proto.core.SourceProto;
import feast.proto.core.StoreProto;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -67,10 +64,6 @@ public static SourceProto.Source getDefaultSource() {
return defaultSource;
}

public static FeatureSetProto.FeatureSet getDefaultFeatureSet() {
return createFeatureSet(DataGenerator.getDefaultSource(), "default", "test");
}

public static SourceProto.Source createSource(String server, String topic) {
return SourceProto.Source.newBuilder()
.setType(SourceProto.SourceType.KAFKA)
Expand Down Expand Up @@ -115,20 +108,6 @@ public static StoreProto.Store createStore(
}
}

public static FeatureSetProto.FeatureSpec createFeature(
String name, ValueProto.ValueType.Enum valueType, Map<String, String> labels) {
return FeatureSetProto.FeatureSpec.newBuilder()
.setName(name)
.setValueType(valueType)
.putAllLabels(labels)
.build();
}

public static FeatureSetProto.EntitySpec createEntitySpec(
String name, ValueProto.ValueType.Enum valueType) {
return FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build();
}

public static EntityProto.EntitySpecV2 createEntitySpecV2(
String name,
String description,
Expand All @@ -151,70 +130,6 @@ public static FeatureProto.FeatureSpecV2 createFeatureSpecV2(
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
List<FeatureSetProto.EntitySpec> entities,
List<FeatureSetProto.FeatureSpec> features,
Map<String, String> labels) {
return FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetProto.FeatureSetSpec.newBuilder()
.setSource(source)
.setName(name)
.setProject(projectName)
.putAllLabels(labels)
.addAllEntities(entities)
.addAllFeatures(features)
.build())
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
Map<String, ValueProto.ValueType.Enum> entities,
Map<String, ValueProto.ValueType.Enum> features,
Map<String, String> labels) {
return FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetProto.FeatureSetSpec.newBuilder()
.setSource(source)
.setName(name)
.setProject(projectName)
.putAllLabels(labels)
.addAllEntities(
entities.entrySet().stream()
.map(entry -> createEntitySpec(entry.getKey(), entry.getValue()))
.collect(Collectors.toList()))
.addAllFeatures(
features.entrySet().stream()
.map(
entry ->
createFeature(
entry.getKey(), entry.getValue(), Collections.emptyMap()))
.collect(Collectors.toList()))
.build())
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
Map<String, ValueProto.ValueType.Enum> entities,
Map<String, ValueProto.ValueType.Enum> features) {
return createFeatureSet(source, projectName, name, entities, features, new HashMap<>());
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source, String projectName, String name) {
return createFeatureSet(
source, projectName, name, Collections.emptyMap(), Collections.emptyMap());
}

// Create a Feature Table spec without DataSources configured.
public static FeatureTableSpec createFeatureTableSpec(
String name,
Expand Down Expand Up @@ -309,6 +224,10 @@ public static ValueProto.Value createEmptyValue() {
return ValueProto.Value.newBuilder().build();
}

public static ValueProto.Value createStrValue(String val) {
return ValueProto.Value.newBuilder().setStringVal(val).build();
}

public static ValueProto.Value createDoubleValue(double value) {
return ValueProto.Value.newBuilder().setDoubleVal(value).build();
}
Expand Down
75 changes: 0 additions & 75 deletions common-test/src/main/java/feast/common/it/SimpleCoreClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,6 @@ public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) {
this.stub = stub;
}

public CoreServiceProto.ApplyFeatureSetResponse simpleApplyFeatureSet(
FeatureSetProto.FeatureSet featureSet) {
return stub.applyFeatureSet(
CoreServiceProto.ApplyFeatureSetRequest.newBuilder().setFeatureSet(featureSet).build());
}

public CoreServiceProto.ApplyEntityResponse simpleApplyEntity(
String projectName, EntityProto.EntitySpecV2 spec) {
return stub.applyEntity(
Expand Down Expand Up @@ -84,53 +78,6 @@ public List<FeatureTableProto.FeatureTable> simpleListFeatureTables(
.getTablesList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName, Map<String, String> labels) {
return stub.listFeatureSets(
CoreServiceProto.ListFeatureSetsRequest.newBuilder()
.setFilter(
CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder()
.setProject(projectName)
.setFeatureSetName(featureSetName)
.putAllLabels(labels)
.build())
.build())
.getFeatureSetsList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName, FeatureSetProto.FeatureSetStatus status) {
return stub.listFeatureSets(
CoreServiceProto.ListFeatureSetsRequest.newBuilder()
.setFilter(
CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder()
.setProject(projectName)
.setFeatureSetName(featureSetName)
.setStatus(status)
.build())
.build())
.getFeatureSetsList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName) {
return simpleListFeatureSets(
projectName, featureSetName, FeatureSetProto.FeatureSetStatus.STATUS_INVALID);
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(String featureSetName) {
return simpleListFeatureSets("default", featureSetName);
}

public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String name) {
return stub.getFeatureSet(
CoreServiceProto.GetFeatureSetRequest.newBuilder()
.setName(name)
.setProject(projectName)
.build())
.getFeatureSet();
}

public EntityProto.Entity simpleGetEntity(String projectName, String name) {
return stub.getEntity(
CoreServiceProto.GetEntityRequest.newBuilder()
Expand All @@ -149,19 +96,6 @@ public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName,
.getTable();
}

public void updateFeatureSetStatus(
String projectName, String name, FeatureSetProto.FeatureSetStatus status) {
stub.updateFeatureSetStatus(
CoreServiceProto.UpdateFeatureSetStatusRequest.newBuilder()
.setReference(
FeatureSetReferenceProto.FeatureSetReference.newBuilder()
.setProject(projectName)
.setName(name)
.build())
.setStatus(status)
.build());
}

public Map<String, FeatureProto.FeatureSpecV2> simpleListFeatures(
String projectName, Map<String, String> labels, List<String> entities) {
return stub.listFeatures(
Expand Down Expand Up @@ -200,15 +134,6 @@ public String getFeastCoreVersion() {
.getVersion();
}

public FeatureSetProto.FeatureSet getFeatureSet(String projectName, String featureSetName) {
return stub.getFeatureSet(
CoreServiceProto.GetFeatureSetRequest.newBuilder()
.setProject(projectName)
.setName(featureSetName)
.build())
.getFeatureSet();
}

public FeatureTableProto.FeatureTable applyFeatureTable(
String projectName, FeatureTableSpec spec) {
return stub.applyFeatureTable(
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/java/feast/common/logging/entry/LogResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
public abstract class LogResource {
public enum ResourceType {
JOB,
FEATURE_SET,
FEATURE_TABLE
}

public abstract ResourceType getType();
Expand Down
34 changes: 19 additions & 15 deletions common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import feast.common.logging.entry.LogResource.ResourceType;
import feast.proto.serving.ServingAPIProto.FeatureReference;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest;
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
import feast.proto.types.ValueProto.Value;
Expand All @@ -34,31 +34,38 @@

public class AuditLogEntryTest {
public List<AuditLogEntry> getTestAuditLogs() {
GetOnlineFeaturesRequest requestSpec =
GetOnlineFeaturesRequest.newBuilder()
.setOmitEntitiesInResponse(false)
GetOnlineFeaturesRequestV2 requestSpec =
GetOnlineFeaturesRequestV2.newBuilder()
.addAllFeatures(
Arrays.asList(
FeatureReference.newBuilder().setName("feature1").build(),
FeatureReference.newBuilder().setName("feature2").build()))
FeatureReferenceV2.newBuilder()
.setFeatureTable("featuretable_1")
.setName("feature1")
.build(),
FeatureReferenceV2.newBuilder()
.setFeatureTable("featuretable_1")
.setName("feature2")
.build()))
.build();

GetOnlineFeaturesResponse responseSpec =
GetOnlineFeaturesResponse.newBuilder()
.addAllFieldValues(
Arrays.asList(
FieldValues.newBuilder()
.putFields("feature", Value.newBuilder().setInt32Val(32).build())
.putFields(
"featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build())
.build(),
FieldValues.newBuilder()
.putFields("feature2", Value.newBuilder().setInt32Val(64).build())
.putFields(
"featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build())
.build()))
.build();

return Arrays.asList(
MessageAuditLogEntry.newBuilder()
.setComponent("feast-serving")
.setVersion("0.6")
.setVersion("0.9")
.setService("ServingService")
.setMethod("getOnlineFeatures")
.setRequest(requestSpec)
Expand All @@ -67,12 +74,9 @@ public List<AuditLogEntry> getTestAuditLogs() {
.setIdentity("adam@no.such.email")
.build(),
ActionAuditLogEntry.of(
"core", "0.6", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
"core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
TransitionAuditLogEntry.of(
"core",
"0.6",
LogResource.of(ResourceType.FEATURE_SET, "project/feature_set"),
"READY"));
"core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY"));
}

@Test
Expand Down
Loading

0 comments on commit 280d87f

Please sign in to comment.