diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 4555fe47ec..57c6970701 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -32,14 +32,11 @@ import feast.proto.core.EntityProto; import feast.proto.core.FeatureProto; import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureSetProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -67,10 +64,6 @@ public static SourceProto.Source getDefaultSource() { return defaultSource; } - public static FeatureSetProto.FeatureSet getDefaultFeatureSet() { - return createFeatureSet(DataGenerator.getDefaultSource(), "default", "test"); - } - public static SourceProto.Source createSource(String server, String topic) { return SourceProto.Source.newBuilder() .setType(SourceProto.SourceType.KAFKA) @@ -115,20 +108,6 @@ public static StoreProto.Store createStore( } } - public static FeatureSetProto.FeatureSpec createFeature( - String name, ValueProto.ValueType.Enum valueType, Map labels) { - return FeatureSetProto.FeatureSpec.newBuilder() - .setName(name) - .setValueType(valueType) - .putAllLabels(labels) - .build(); - } - - public static FeatureSetProto.EntitySpec createEntitySpec( - String name, ValueProto.ValueType.Enum valueType) { - return FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build(); - } - public static EntityProto.EntitySpecV2 createEntitySpecV2( String name, String description, @@ -151,70 +130,6 @@ public static FeatureProto.FeatureSpecV2 createFeatureSpecV2( .build(); } - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - List entities, - List features, - Map labels) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .putAllLabels(labels) - .addAllEntities(entities) - .addAllFeatures(features) - .build()) - .build(); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - Map entities, - Map features, - Map labels) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .putAllLabels(labels) - .addAllEntities( - entities.entrySet().stream() - .map(entry -> createEntitySpec(entry.getKey(), entry.getValue())) - .collect(Collectors.toList())) - .addAllFeatures( - features.entrySet().stream() - .map( - entry -> - createFeature( - entry.getKey(), entry.getValue(), Collections.emptyMap())) - .collect(Collectors.toList())) - .build()) - .build(); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - Map entities, - Map features) { - return createFeatureSet(source, projectName, name, entities, features, new HashMap<>()); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, String projectName, String name) { - return createFeatureSet( - source, projectName, name, Collections.emptyMap(), Collections.emptyMap()); - } - // Create a Feature Table spec without DataSources configured. public static FeatureTableSpec createFeatureTableSpec( String name, @@ -309,6 +224,10 @@ public static ValueProto.Value createEmptyValue() { return ValueProto.Value.newBuilder().build(); } + public static ValueProto.Value createStrValue(String val) { + return ValueProto.Value.newBuilder().setStringVal(val).build(); + } + public static ValueProto.Value createDoubleValue(double value) { return ValueProto.Value.newBuilder().setDoubleVal(value).build(); } diff --git a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java index 3c3bfbc553..11fa6715dc 100644 --- a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java @@ -31,12 +31,6 @@ public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { this.stub = stub; } - public CoreServiceProto.ApplyFeatureSetResponse simpleApplyFeatureSet( - FeatureSetProto.FeatureSet featureSet) { - return stub.applyFeatureSet( - CoreServiceProto.ApplyFeatureSetRequest.newBuilder().setFeatureSet(featureSet).build()); - } - public CoreServiceProto.ApplyEntityResponse simpleApplyEntity( String projectName, EntityProto.EntitySpecV2 spec) { return stub.applyEntity( @@ -84,53 +78,6 @@ public List simpleListFeatureTables( .getTablesList(); } - public List simpleListFeatureSets( - String projectName, String featureSetName, Map labels) { - return stub.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(projectName) - .setFeatureSetName(featureSetName) - .putAllLabels(labels) - .build()) - .build()) - .getFeatureSetsList(); - } - - public List simpleListFeatureSets( - String projectName, String featureSetName, FeatureSetProto.FeatureSetStatus status) { - return stub.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(projectName) - .setFeatureSetName(featureSetName) - .setStatus(status) - .build()) - .build()) - .getFeatureSetsList(); - } - - public List simpleListFeatureSets( - String projectName, String featureSetName) { - return simpleListFeatureSets( - projectName, featureSetName, FeatureSetProto.FeatureSetStatus.STATUS_INVALID); - } - - public List simpleListFeatureSets(String featureSetName) { - return simpleListFeatureSets("default", featureSetName); - } - - public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String name) { - return stub.getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getFeatureSet(); - } - public EntityProto.Entity simpleGetEntity(String projectName, String name) { return stub.getEntity( CoreServiceProto.GetEntityRequest.newBuilder() @@ -149,19 +96,6 @@ public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, .getTable(); } - public void updateFeatureSetStatus( - String projectName, String name, FeatureSetProto.FeatureSetStatus status) { - stub.updateFeatureSetStatus( - CoreServiceProto.UpdateFeatureSetStatusRequest.newBuilder() - .setReference( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setProject(projectName) - .setName(name) - .build()) - .setStatus(status) - .build()); - } - public Map simpleListFeatures( String projectName, Map labels, List entities) { return stub.listFeatures( @@ -200,15 +134,6 @@ public String getFeastCoreVersion() { .getVersion(); } - public FeatureSetProto.FeatureSet getFeatureSet(String projectName, String featureSetName) { - return stub.getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setProject(projectName) - .setName(featureSetName) - .build()) - .getFeatureSet(); - } - public FeatureTableProto.FeatureTable applyFeatureTable( String projectName, FeatureTableSpec spec) { return stub.applyFeatureTable( diff --git a/common/src/main/java/feast/common/logging/entry/LogResource.java b/common/src/main/java/feast/common/logging/entry/LogResource.java index 02e7589f97..1d0345a404 100644 --- a/common/src/main/java/feast/common/logging/entry/LogResource.java +++ b/common/src/main/java/feast/common/logging/entry/LogResource.java @@ -26,7 +26,7 @@ public abstract class LogResource { public enum ResourceType { JOB, - FEATURE_SET, + FEATURE_TABLE } public abstract ResourceType getType(); diff --git a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java b/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java index a332e0be79..cf355e09e4 100644 --- a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java +++ b/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java @@ -22,8 +22,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import feast.common.logging.entry.LogResource.ResourceType; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto.Value; @@ -34,13 +34,18 @@ public class AuditLogEntryTest { public List getTestAuditLogs() { - GetOnlineFeaturesRequest requestSpec = - GetOnlineFeaturesRequest.newBuilder() - .setOmitEntitiesInResponse(false) + GetOnlineFeaturesRequestV2 requestSpec = + GetOnlineFeaturesRequestV2.newBuilder() .addAllFeatures( Arrays.asList( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").build())) + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature1") + .build(), + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature2") + .build())) .build(); GetOnlineFeaturesResponse responseSpec = @@ -48,17 +53,19 @@ public List getTestAuditLogs() { .addAllFieldValues( Arrays.asList( FieldValues.newBuilder() - .putFields("feature", Value.newBuilder().setInt32Val(32).build()) + .putFields( + "featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build()) .build(), FieldValues.newBuilder() - .putFields("feature2", Value.newBuilder().setInt32Val(64).build()) + .putFields( + "featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build()) .build())) .build(); return Arrays.asList( MessageAuditLogEntry.newBuilder() .setComponent("feast-serving") - .setVersion("0.6") + .setVersion("0.9") .setService("ServingService") .setMethod("getOnlineFeatures") .setRequest(requestSpec) @@ -67,12 +74,9 @@ public List getTestAuditLogs() { .setIdentity("adam@no.such.email") .build(), ActionAuditLogEntry.of( - "core", "0.6", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"), + "core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"), TransitionAuditLogEntry.of( - "core", - "0.6", - LogResource.of(ResourceType.FEATURE_SET, "project/feature_set"), - "READY")); + "core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY")); } @Test diff --git a/common/src/test/java/feast/common/models/FeatureSetTest.java b/common/src/test/java/feast/common/models/FeatureSetTest.java deleted file mode 100644 index 52b7dd36ad..0000000000 --- a/common/src/test/java/feast/common/models/FeatureSetTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; - -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.types.ValueProto; -import java.util.Arrays; -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.tensorflow.metadata.v0.*; - -public class FeatureSetTest { - - private List entitySpecs; - private List featureSpecs; - - @Before - public void setUp() { - // Entity Specs - EntitySpec entitySpec1 = - EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - EntitySpec entitySpec2 = - EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - - // Feature Specs - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") - .build(); - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build(); - - entitySpecs = Arrays.asList(entitySpec1, entitySpec2); - featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - } - - @Test - public void shouldReturnFeatureSetStringRef() { - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - - FeatureSetReferenceProto.FeatureSetReference featureSetReference = - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setName(featureSetSpec.getName()) - .setProject(featureSetSpec.getProject()) - .build(); - - String actualFeatureSetStringRef1 = FeatureSet.getFeatureSetStringRef(featureSetSpec); - String actualFeatureSetStringRef2 = FeatureSet.getFeatureSetStringRef(featureSetReference); - String expectedFeatureSetStringRef = "project1/featureSetWithConstraints"; - - assertThat(actualFeatureSetStringRef1, equalTo(expectedFeatureSetStringRef)); - assertThat(actualFeatureSetStringRef2, equalTo(expectedFeatureSetStringRef)); - } -} diff --git a/common/src/test/java/feast/common/models/FeaturesTest.java b/common/src/test/java/feast/common/models/FeaturesTest.java index a4426ad21c..180f7e4e69 100644 --- a/common/src/test/java/feast/common/models/FeaturesTest.java +++ b/common/src/test/java/feast/common/models/FeaturesTest.java @@ -19,81 +19,28 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto; -import feast.proto.types.ValueProto; -import java.util.Arrays; -import java.util.List; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import org.junit.Before; import org.junit.Test; -import org.tensorflow.metadata.v0.*; public class FeaturesTest { - private List entitySpecs; - private List featureSpecs; + private FeatureReferenceV2 featureReference; @Before public void setUp() { - // Entity Specs - EntitySpec entitySpec1 = - EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - EntitySpec entitySpec2 = - EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - - // Feature Specs - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() + featureReference = + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") .build(); - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build(); - - entitySpecs = Arrays.asList(entitySpec1, entitySpec2); - featureSpecs = Arrays.asList(featureSpec1, featureSpec2); } @Test public void shouldReturnFeatureStringRef() { - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - - ServingAPIProto.FeatureReference featureReference = - ServingAPIProto.FeatureReference.newBuilder() - .setProject(featureSetSpec.getProject()) - .setFeatureSet(featureSetSpec.getName()) - .setName(featureSetSpec.getFeatures(0).getName()) - .build(); - - String actualFeatureStringRef = Feature.getFeatureStringWithProjectRef(featureReference); - String actualFeatureIgnoreProjectStringRef = Feature.getFeatureStringRef(featureReference); - String expectedFeatureStringRef = "project1/featureSetWithConstraints:feature1"; - String expectedFeatureIgnoreProjectStringRef = "featureSetWithConstraints:feature1"; + String actualFeatureStringRef = FeatureV2.getFeatureStringRef(featureReference); + String expectedFeatureStringRef = "featuretable_1:feature1"; assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef)); - assertThat(actualFeatureIgnoreProjectStringRef, equalTo(expectedFeatureIgnoreProjectStringRef)); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java index 3de03807c4..4bfa084e15 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java @@ -23,33 +23,23 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.google.protobuf.InvalidProtocolBufferException; import feast.common.auth.authorization.AuthorizationProvider; import feast.common.auth.authorization.AuthorizationResult; import feast.common.auth.config.SecurityProperties; import feast.common.auth.service.AuthorizationService; +import feast.common.it.DataGenerator; import feast.core.config.FeastProperties; import feast.core.dao.ProjectRepository; import feast.core.grpc.CoreServiceImpl; -import feast.core.model.Entity; -import feast.core.model.Feature; -import feast.core.model.FeatureSet; -import feast.core.model.Source; import feast.core.service.ProjectService; import feast.core.service.SpecService; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetRequest; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.types.ValueProto.ValueType.Enum; +import feast.proto.core.CoreServiceProto.ApplyEntityRequest; +import feast.proto.core.CoreServiceProto.ApplyEntityResponse; +import feast.proto.core.EntityProto; +import feast.proto.types.ValueProto; import io.grpc.internal.testing.StreamRecorder; -import java.sql.Date; -import java.time.Instant; -import java.util.Arrays; -import java.util.HashMap; import org.junit.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -82,7 +72,7 @@ public CoreServiceAuthTest() { } @Test - public void shouldNotApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferException { + public void shouldNotApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { String project = "project1"; Authentication auth = mock(Authentication.class); @@ -94,22 +84,23 @@ public void shouldNotApplyFeatureSetIfNotProjectMember() throws InvalidProtocolB .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - StreamRecorder responseObserver = StreamRecorder.create(); - FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", 1, project).toProto(); + StreamRecorder responseObserver = StreamRecorder.create(); + EntityProto.EntitySpecV2 incomingEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec = - incomingFeatureSet.getSpec().toBuilder().build(); - FeatureSetProto.FeatureSet spec = - FeatureSetProto.FeatureSet.newBuilder().setSpec(incomingFeatureSetSpec).build(); - ApplyFeatureSetRequest request = - ApplyFeatureSetRequest.newBuilder().setFeatureSet(spec).build(); + ApplyEntityRequest request = + ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - coreService.applyFeatureSet(request, responseObserver); + coreService.applyEntity(request, responseObserver); assertEquals("PERMISSION_DENIED: Access Denied", responseObserver.getError().getMessage()); } @Test - public void shouldApplyFeatureSetIfProjectMember() throws InvalidProtocolBufferException { + public void shouldApplyEntityIfProjectMember() throws InvalidProtocolBufferException { String project = "project1"; Authentication auth = mock(Authentication.class); @@ -120,42 +111,16 @@ public void shouldApplyFeatureSetIfProjectMember() throws InvalidProtocolBufferE .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - StreamRecorder responseObserver = StreamRecorder.create(); - FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", 1, project).toProto(); - FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec = - incomingFeatureSet.getSpec().toBuilder().build(); - FeatureSetProto.FeatureSet spec = - FeatureSetProto.FeatureSet.newBuilder().setSpec(incomingFeatureSetSpec).build(); - ApplyFeatureSetRequest request = - ApplyFeatureSetRequest.newBuilder().setFeatureSet(spec).build(); - - coreService.applyFeatureSet(request, responseObserver); - } - - private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Feature feature = new Feature("feature", Enum.INT64); - Entity entity = new Entity("entity", Enum.STRING); - SourceProto.Source sourceSpec = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("my-topic") - .build()) - .build(); - Source defaultSource = Source.fromProto(sourceSpec); - FeatureSet fs = - new FeatureSet( - name, - project, - 100L, - Arrays.asList(entity), - Arrays.asList(feature), - defaultSource, - new HashMap(), - FeatureSetStatus.STATUS_READY); - fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); - return fs; + StreamRecorder responseObserver = StreamRecorder.create(); + EntityProto.EntitySpecV2 incomingEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + ApplyEntityRequest request = + ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); + + coreService.applyEntity(request, responseObserver); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index d6f13fdb55..77cae43cd8 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.nimbusds.jose.JOSEException; @@ -28,6 +29,7 @@ import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; import feast.proto.core.*; +import feast.proto.types.ValueProto; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; @@ -57,8 +59,6 @@ public class CoreServiceAuthenticationIT extends BaseIT { private static JwtHelper jwtHelper = new JwtHelper(); - static String project = "myproject"; - static String subject = "random@example.com"; static String subjectClaim = "sub"; @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); @@ -126,54 +126,40 @@ public void shouldGetVersionFromFeastCoreAlways() { * Core as anonymous users. They are not forced to authenticate. */ @Test - public void shouldAllowUnauthenticatedFeatureSetListing() { - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - insecureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - - List listFeatureSetsResponse = - insecureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); + public void shouldAllowUnauthenticatedEntityApplyAndListing() { + String project = "default"; + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + insecureApiClient.simpleApplyEntity(project, expectedEntitySpec); + + List listEntitiesResponse = insecureApiClient.simpleListEntities(project); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @Test - public void shouldAllowAuthenticatedFeatureSetListing() { + public void shouldAllowAuthenticatedEntityApplyAndListing() { SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - List listFeatureSetsResponse = - secureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - } - - @Test - void canApplyFeatureSetIfAuthenticated() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "project_1", "test_1"); - - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet("project_1", "test_1"); - - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + String project = "default"; + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); + List listEntitiesResponse = insecureApiClient.simpleListEntities(project); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @TestConfiguration diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java index 584fcc3854..41faee7f71 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.google.protobuf.InvalidProtocolBufferException; @@ -30,7 +31,8 @@ import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.FeatureSetProto; +import feast.proto.core.EntityProto; +import feast.proto.types.ValueProto; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; @@ -153,8 +155,13 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { @BeforeEach public void setUp() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); } @AfterAll @@ -176,12 +183,12 @@ public void shouldGetVersionFromFeastCoreAlways() { } @Test - public void shouldNotAllowUnauthenticatedFeatureSetListing() { + public void shouldNotAllowUnauthenticatedEntityListing() { Exception exception = assertThrows( StatusRuntimeException.class, () -> { - insecureApiClient.simpleListFeatureSets("8"); + insecureApiClient.simpleListEntities("8"); }); String expectedMessage = "UNAUTHENTICATED: Authentication failed"; @@ -190,32 +197,37 @@ public void shouldNotAllowUnauthenticatedFeatureSetListing() { } @Test - public void shouldAllowAuthenticatedFeatureSetListing() { + public void shouldAllowAuthenticatedEntityListing() { SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - List listFeatureSetsResponse = - secureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @Test - void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferException { + void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { String userName = "random_user@example.com"; SimpleCoreClient secureApiClient = getSecureApiClient(userName); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_5"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); StatusRuntimeException exception = assertThrows( StatusRuntimeException.class, - () -> secureApiClient.simpleApplyFeatureSet(expectedFeatureSet)); + () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); String expectedMessage = String.format( @@ -225,37 +237,39 @@ void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferExcepti } @Test - void canApplyFeatureSetIfProjectMember() { + void canApplyEntityIfProjectMember() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_6"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity_6", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet(project, "test_6"); + EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); + assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); } @Test - void canApplyFeatureSetIfAdmin() { + void canApplyEntityIfAdmin() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "any_project", "test_2"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity_7", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet("any_project", "test_2"); + EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); + assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); } @TestConfiguration diff --git a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java index 0297060feb..dbad956e5b 100644 --- a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java +++ b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java @@ -25,13 +25,10 @@ import feast.common.it.BaseIT; import feast.common.it.DataGenerator; import feast.common.it.SimpleCoreClient; -import feast.core.model.Project; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.EntityProto; -import feast.proto.core.FeatureSetProto.FeatureSet; import feast.proto.core.FeatureTableProto; import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.ValueType.Enum; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.restassured.RestAssured; @@ -98,80 +95,7 @@ public void listProjects() { .getBody() .asString(); List projectList = JsonPath.from(responseBody).getList("projects"); - assertEquals(projectList, List.of("default", "merchant")); - } - - // list feature sets - @Test - public void listFeatureSets() { - // project = default - // name = merchant_ratings - // getting a specific feature set - String uri1 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "default") - .queryParam("name", "merchant_ratings") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 1); - - // project = * - // name = *merchant_ratings - // should have two feature sets named *merchant_ratings - String uri2 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "*") - .queryParam("name", "*merchant_ratings") - .buildAndExpand() - .toString(); - responseBody = - get(uri2) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 2); - - // project = * - // name = * - // should have three feature sets - String uri3 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "*") - .queryParam("name", "*") - .buildAndExpand() - .toString(); - responseBody = - get(uri3) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 3); + assertEquals(projectList, List.of("default")); } @Test @@ -250,36 +174,6 @@ public void listFeatureTables() { @BeforeEach private void createSpecs() { - // Apply feature sets - FeatureSet merchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - Project.DEFAULT_NAME, - "merchant_ratings", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of("average_rating", Enum.DOUBLE, "total_ratings", Enum.INT64)); - apiClient.simpleApplyFeatureSet(merchantFeatureSet); - - FeatureSet anotherMerchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - Project.DEFAULT_NAME, - "another_merchant_ratings", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of( - "another_average_rating", Enum.DOUBLE, - "another_total_ratings", Enum.INT64)); - apiClient.simpleApplyFeatureSet(anotherMerchantFeatureSet); - - FeatureSet yetAnotherMerchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "merchant", - "yet_another_merchant_feature_set", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of("merchant_prop1", Enum.BOOL, "merchant_prop2", Enum.FLOAT)); - apiClient.simpleApplyFeatureSet(yetAnotherMerchantFeatureSet); - // Apply entities EntityProto.EntitySpecV2 entitySpec1 = DataGenerator.createEntitySpecV2( diff --git a/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/core/src/test/java/feast/core/logging/CoreLoggingIT.java index 0ca9eda32c..0f137b4639 100644 --- a/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ b/core/src/test/java/feast/core/logging/CoreLoggingIT.java @@ -35,7 +35,7 @@ import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListStoresRequest; import feast.proto.core.CoreServiceProto.ListStoresResponse; import feast.proto.core.CoreServiceProto.UpdateStoreRequest; @@ -123,19 +123,15 @@ public void shouldProduceMessageAuditLogsOnCall() @Test public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { // Send a bad request which should cause Core to error - ListFeatureSetsRequest request = - ListFeatureSetsRequest.newBuilder() - .setFilter( - ListFeatureSetsRequest.Filter.newBuilder() - .setProject("*") - .setFeatureSetName("nop") - .build()) + ListFeatureTablesRequest request = + ListFeatureTablesRequest.newBuilder() + .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) .build(); boolean hasExpectedException = false; Code statusCode = null; try { - coreService.listFeatureSets(request); + coreService.listFeatureTables(request); } catch (StatusRuntimeException e) { hasExpectedException = true; statusCode = e.getStatus().getCode(); @@ -146,7 +142,7 @@ public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { Thread.sleep(1000); // Pull message audit logs logs from test log appender List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureSets"); + parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); assertEquals(1, logJsonObjects.size()); JsonObject logJsonObject = logJsonObjects.get(0); diff --git a/core/src/test/java/feast/core/model/FeatureSetTest.java b/core/src/test/java/feast/core/model/FeatureSetTest.java deleted file mode 100644 index 270dc3f3bc..0000000000 --- a/core/src/test/java/feast/core/model/FeatureSetTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -import com.google.protobuf.Duration; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.types.ValueProto.ValueType.Enum; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.tensorflow.metadata.v0.IntDomain; - -public class FeatureSetTest { - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - private FeatureSetProto.FeatureSet oldFeatureSetProto; - - @Before - public void setUp() { - SourceProto.Source oldSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic")) - .build(); - - oldFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(100)) - .setSource(oldSource) - .addFeatures( - FeatureSpec.newBuilder().setName("feature1").setValueType(Enum.INT64)) - .addFeatures( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - } - - @Test - public void shouldUpdateFromProto() throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - - FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); - actual.updateFromProto(newFeatureSetProto); - - FeatureSet expected = FeatureSet.fromProto(newFeatureSetProto); - Feature archivedFeature = - Feature.fromProto( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING).build()); - archivedFeature.setArchived(true); - expected.addFeature(archivedFeature); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldNotUpdateIfNoChange() throws InvalidProtocolBufferException { - FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); - actual.setStatus(FeatureSetStatus.STATUS_READY); - actual.updateFromProto(oldFeatureSetProto); - - FeatureSet expected = FeatureSet.fromProto(oldFeatureSetProto); - expected.setStatus(FeatureSetStatus.STATUS_READY); - - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldThrowExceptionIfUpdateWithEntitiesChanged() - throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) - .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.FLOAT)) - .build()) - .build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("does not match existing set of entities")); - FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); - existingFeatureSet.updateFromProto(newFeatureSetProto); - } - - @Test - public void shouldThrowExceptionIfUpdateWithFeatureTypesChanged() - throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.FLOAT)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - containsString( - "You are attempting to change the type of feature feature2 from STRING to FLOAT.")); - FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); - existingFeatureSet.updateFromProto(newFeatureSetProto); - } -} diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 7f0b88539c..cee48f1dd0 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -17,20 +17,16 @@ package feast.core.service; import static com.jayway.jsonassert.impl.matcher.IsMapContainingKey.hasKey; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsMapContaining.hasEntry; import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsIterableContaining.hasItem; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; import avro.shaded.com.google.common.collect.ImmutableMap; -import com.google.protobuf.Duration; import feast.common.it.BaseIT; import feast.common.it.DataGenerator; import feast.common.it.SimpleCoreClient; @@ -42,7 +38,6 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import java.util.*; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -50,7 +45,6 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; -import org.tensorflow.metadata.v0.*; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; @SpringBootTest @@ -126,137 +120,9 @@ public void initState() { "Entity 3 description", ValueProto.ValueType.Enum.STRING, ImmutableMap.of("label_key2", "label_value2"))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "default", - "fs1", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "default", - "fs2", - ImmutableMap.of("user_id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("sum", ValueProto.ValueType.Enum.INT64))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs3", - ImmutableList.of( - DataGenerator.createEntitySpec("user_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature1", ValueProto.ValueType.Enum.INT32, Collections.emptyMap()), - DataGenerator.createFeature( - "feature2", ValueProto.ValueType.Enum.INT32, Collections.emptyMap())), - Collections.emptyMap())); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs4", - ImmutableList.of( - DataGenerator.createEntitySpec("customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature2", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "one"))), - ImmutableMap.of("label", "some"))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs5", - ImmutableList.of( - DataGenerator.createEntitySpec("customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature3", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "two"))), - Collections.emptyMap())); - apiClient.simpleApplyFeatureSet(DataGenerator.createFeatureSet(source, "default", "new_fs")); apiClient.updateStore(DataGenerator.getDefaultStore()); } - @Nested - class ListFeatureSets { - - @Test - public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided() { - List featureSets = apiClient.simpleListFeatureSets("*", "*"); - - assertThat(featureSets, hasSize(6)); - } - - @Test - public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() { - List featureSets = - apiClient.simpleListFeatureSets("default", "fs*"); - - assertThat(featureSets, hasSize(2)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs1"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs2"))))); - } - - @Test - public void shouldFilterFeatureSetsByNameAndProject() { - List featureSets = - apiClient.simpleListFeatureSets("project1", "fs3"); - - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs3"))))); - } - - @Test - public void shouldFilterFeatureSetsByStatus() { - apiClient.updateFeatureSetStatus( - "project1", "fs3", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - apiClient.updateFeatureSetStatus( - "project1", "fs4", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - List featureSets = - apiClient.simpleListFeatureSets("*", "*", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - assertThat(featureSets, hasSize(2)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs3"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs4"))))); - - assertThat( - apiClient.simpleListFeatureSets( - "default", "*", FeatureSetProto.FeatureSetStatus.STATUS_PENDING), - hasSize(3)); - } - - @Test - public void shouldFilterFeatureSetsByLabels() { - List featureSets = - apiClient.simpleListFeatureSets("project1", "*", ImmutableMap.of("label", "some")); - - assertThat(featureSets, hasSize(1)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs4"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - List featureSets = apiClient.simpleListFeatureSets("", "*"); - - assertThat(featureSets, hasSize(3)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs1"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs2"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("new_fs"))))); - } - - @Test - public void shouldThrowExceptionGivenMissingFeatureSetName() { - assertThrows(StatusRuntimeException.class, () -> apiClient.simpleListFeatureSets("", "")); - } - } - @Nested class ListEntities { @Test @@ -358,393 +224,6 @@ public void shouldThrowExceptionGivenWildcardProject() { } } - @Nested - class ApplyFeatureSet { - @Test - public void shouldThrowExceptionGivenReservedFeatureName() { - List reservedNames = - Arrays.asList("created_timestamp", "event_timestamp", "ingestion_id", "job_id"); - String reservedNamesString = StringUtils.join(reservedNames, ", "); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project", - "name", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("event_timestamp", ValueProto.ValueType.Enum.STRING)))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: Reserved feature names have been used, which are not allowed. These names include %s." - + "You've just used an invalid name, %s.", - reservedNamesString, "event_timestamp"))); - } - - @Test - public void shouldThrowExceptionGivenFeatureSetWithDash() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project", - "dash-name", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("test_string", ValueProto.ValueType.Enum.STRING)))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: invalid value for %s resource, %s: %s", - "featureset", - "dash-name", - "argument must only contain alphanumeric characters and underscores."))); - } - - @Test - public void shouldReturnFeatureSetIfFeatureSetHasNotChanged() { - FeatureSetProto.FeatureSet featureSet = apiClient.getFeatureSet("default", "fs1"); - - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet(featureSet); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.NO_CHANGE)); - assertThat( - response.getFeatureSet().getSpec().getVersion(), - equalTo(featureSet.getSpec().getVersion())); - } - - @Test - public void shouldApplyFeatureSetIfNotExists() { - FeatureSetProto.FeatureSet featureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "new", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("feature", ValueProto.ValueType.Enum.STRING)); - - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet(featureSet); - - assertThat( - response.getFeatureSet().getSpec(), - equalTo( - featureSet - .getSpec() - .toBuilder() - .setVersion(1) - .setMaxAge(Duration.newBuilder().build()) - .build())); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - } - - @Test - public void shouldUpdateAndSaveFeatureSetIfAlreadyExists() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "fs1", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of( - "total", ValueProto.ValueType.Enum.INT64, - "subtotal", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem(hasProperty("name", equalTo("subtotal")))); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat(response.getFeatureSet().getSpec().getVersion(), equalTo(2)); - } - - @Test - public void shouldAcceptPresenceShapeAndDomainConstraints() { - List entitySpecs = new ArrayList<>(); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity3") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity4") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity5") - .setValueType(ValueProto.ValueType.Enum.BOOL) - .build()); - - List featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature3") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setFloatDomain(FloatDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature4") - .setValueType(ValueProto.ValueType.Enum.STRING) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setStringDomain(StringDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature5") - .setValueType(ValueProto.ValueType.Enum.BOOL) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setBoolDomain(BoolDomain.getDefaultInstance()) - .build()); - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - CoreServiceProto.ApplyFeatureSetResponse applyFeatureSetResponse = - apiClient.simpleApplyFeatureSet(featureSet); - FeatureSetProto.FeatureSetSpec appliedFeatureSetSpec = - applyFeatureSetResponse.getFeatureSet().getSpec(); - - // appliedEntitySpecs needs to be sorted because the list returned by specService may not - // follow the order in the request - List appliedEntitySpecs = - new ArrayList<>(appliedFeatureSetSpec.getEntitiesList()); - appliedEntitySpecs.sort(Comparator.comparing(FeatureSetProto.EntitySpec::getName)); - - // appliedFeatureSpecs needs to be sorted because the list returned by specService may not - // follow the order in the request - List appliedFeatureSpecs = - new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); - appliedFeatureSpecs.sort(Comparator.comparing(FeatureSetProto.FeatureSpec::getName)); - - assertEquals(appliedEntitySpecs, entitySpecs); - assertEquals(appliedFeatureSpecs, featureSpecs); - } - - @Test - public void shouldUpdateFeatureSetWhenConstraintsAreUpdated() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "fs1", - ImmutableList.of( - FeatureSetProto.EntitySpec.newBuilder() - .setName("id") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()), - ImmutableList.of( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("total") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMin(0).setMax(100).build()) - .build()), - Collections.emptyMap())); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - hasProperty( - "intDomain", equalTo(IntDomain.newBuilder().setMin(0).setMax(100).build())))); - } - - @Test - public void shouldCreateProjectWhenNotAlreadyExists() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "new_project", - "new_fs", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getProject(), equalTo("new_project")); - } - - @Test - public void shouldUsedDefaultProjectIfUnspecified() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "", - "some", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getProject(), equalTo("default")); - } - - @Test - public void shouldFailWhenProjectIsArchived() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "archived", - "fs", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64)))); - assertThat(exc.getMessage(), equalTo("INTERNAL: Project is archived: archived")); - } - - @Test - public void shouldAcceptFeatureLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "some", - ImmutableList.of( - FeatureSetProto.EntitySpec.newBuilder() - .setName("id") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()), - ImmutableList.of( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .putAllLabels(ImmutableMap.of("type", "integer")) - .build(), - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.STRING) - .putAllLabels(ImmutableMap.of("type", "string")) - .build()), - Collections.emptyMap())); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature1")), - hasProperty("labelsMap", hasEntry("type", "integer"))))); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature2")), - hasProperty("labelsMap", hasEntry("type", "string"))))); - } - - @Test - public void shouldUpdateLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project1", - "fs4", - ImmutableList.of( - DataGenerator.createEntitySpec( - "customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature2", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "two"))), - ImmutableMap.of("label", "some"))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature2")), - hasProperty("labelsMap", hasEntry("version", "two"))))); - } - - @Test - public void shouldAcceptFeatureSetLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "", - "some", - ImmutableList.of( - DataGenerator.createEntitySpec( - "customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of(), - ImmutableMap.of("label", "some"))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getLabelsMap(), hasEntry("label", "some")); - } - } - @Nested class ApplyEntity { @Test @@ -904,20 +383,6 @@ public void shouldDoNothingIfNoChange() { } } - @Nested - class GetFeatureSet { - @Test - public void shouldThrowExceptionGivenMissingFeatureSet() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.getFeatureSet("default", "unknown")); - - assertThat( - exc.getMessage(), - equalTo("INTERNAL: Feature set with name \"unknown\" could not be found.")); - } - } - @Nested class GetEntity { @Test diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java deleted file mode 100644 index 155a52d100..0000000000 --- a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import feast.proto.core.FeatureSetProto; -import feast.proto.types.ValueProto; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -public class FeatureSetValidatorTest { - - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - @Test - public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() { - Map featureLabels = - new HashMap<>() { - { - put("", "empty_key"); - } - }; - - List featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .putAllLabels(featureLabels) - .build()); - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllFeatures(featureSpecs) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Feature label keys must not be empty"); - FeatureSetValidator.validateSpec(featureSet); - } - - @Test - public void shouldThrowExceptionForFeatureSetLabelsWithAnEmptyKey() { - - Map featureSetLabels = - new HashMap<>() { - { - put("", "empty_key"); - } - }; - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .putAllLabels(featureSetLabels) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Feature set label keys must not be empty"); - FeatureSetValidator.validateSpec(featureSet); - } -} diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java index fdf4d0469d..17332123d2 100644 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ b/core/src/test/java/feast/core/validators/MatchersTest.java @@ -31,13 +31,13 @@ public class MatchersTest { @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { String in = "REDIS_DB"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { String in = "REDIS1"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test @@ -46,11 +46,11 @@ public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForIn exception.expectMessage( Strings.lenientFormat( "invalid value for %s resource, %s: %s", - "featureset", + "featuretable", "redis", "argument must be in upper snake case, and cannot include any special characters.")); String in = "redis"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test diff --git a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java index 671b57b2f6..b824f3ae43 100644 --- a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java +++ b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java @@ -30,15 +30,14 @@ import feast.common.auth.config.SecurityProperties.AuthenticationProperties; import feast.common.auth.config.SecurityProperties.AuthorizationProperties; import feast.common.auth.service.AuthorizationService; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.types.ValueProto.Value; import feast.serving.config.FeastProperties; import feast.serving.service.ServingService; import feast.serving.service.ServingServiceV2; -import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.jaegertracing.Configuration; import io.opentracing.Tracer; @@ -58,7 +57,7 @@ public class ServingServiceGRpcControllerTest { @Mock private StreamObserver mockStreamObserver; - private GetOnlineFeaturesRequest validRequest; + private GetOnlineFeaturesRequestV2 validRequest; private ServingServiceGRpcController service; @@ -71,12 +70,20 @@ public void setUp() { initMocks(this); validRequest = - GetOnlineFeaturesRequest.newBuilder() - .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + GetOnlineFeaturesRequestV2.newBuilder() + .addFeatures( + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature1") + .build()) + .addFeatures( + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature2") + .build()) .addEntityRows( EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields("entity1", Value.newBuilder().setInt64Val(1).build()) .putFields("entity2", Value.newBuilder().setInt64Val(1).build())) .build(); @@ -103,17 +110,17 @@ private ServingServiceGRpcController getServingServiceGRpcController(boolean ena @Test public void shouldPassValidRequestAsIs() { service = getServingServiceGRpcController(false); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockServingService).getOnlineFeatures(validRequest); + service.getOnlineFeaturesV2(validRequest, mockStreamObserver); + Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); } @Test public void shouldCallOnErrorIfEntityDatasetIsNotSet() { service = getServingServiceGRpcController(false); - GetOnlineFeaturesRequest missingEntityName = - GetOnlineFeaturesRequest.newBuilder(validRequest).clearEntityRows().build(); - service.getOnlineFeatures(missingEntityName, mockStreamObserver); - Mockito.verify(mockStreamObserver).onError(Mockito.any(StatusRuntimeException.class)); + GetOnlineFeaturesRequestV2 missingEntityName = + GetOnlineFeaturesRequestV2.newBuilder(validRequest).clearEntityRows().build(); + service.getOnlineFeaturesV2(missingEntityName, mockStreamObserver); + Mockito.verify(mockStreamObserver).onError(Mockito.any(IllegalArgumentException.class)); } @Test @@ -125,20 +132,7 @@ public void shouldPassValidRequestAsIsIfRequestIsAuthorized() { doReturn(AuthorizationResult.success()) .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockServingService).getOnlineFeatures(validRequest); - } - - @Test - public void shouldThrowErrorOnValidRequestIfRequestIsUnauthorized() { - service = getServingServiceGRpcController(true); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(authentication); - doReturn(AuthorizationResult.failed(null)) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockStreamObserver).onError(Mockito.any(StatusRuntimeException.class)); + service.getOnlineFeaturesV2(validRequest, mockStreamObserver); + Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); } } diff --git a/serving/src/test/java/feast/serving/it/AuthTestUtils.java b/serving/src/test/java/feast/serving/it/AuthTestUtils.java index e5e0def801..1990ad4ece 100644 --- a/serving/src/test/java/feast/serving/it/AuthTestUtils.java +++ b/serving/src/test/java/feast/serving/it/AuthTestUtils.java @@ -16,42 +16,28 @@ */ package feast.serving.it; -import static org.awaitility.Awaitility.waitAtMost; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.protobuf.Timestamp; import feast.common.auth.credentials.OAuthCredentials; import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; import feast.proto.core.SourceProto; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import org.apache.commons.lang3.tuple.Pair; import org.junit.runners.model.InitializationError; import sh.ory.keto.ApiClient; import sh.ory.keto.ApiException; @@ -82,81 +68,26 @@ public static SourceProto.Source createSource(String server, String topic) { .build(); } - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, + public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( String projectName, - String name, - List> entities, - List> features) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .addAllEntities( - entities.stream() - .map( - pair -> - FeatureSetProto.EntitySpec.newBuilder() - .setName(pair.getLeft()) - .setValueType(pair.getRight()) - .build()) - .collect(Collectors.toList())) - .addAllFeatures( - features.stream() - .map( - pair -> - FeatureSetProto.FeatureSpec.newBuilder() - .setName(pair.getLeft()) - .setValueType(pair.getRight()) - .build()) - .collect(Collectors.toList())) - .build()) - .build(); - } - - public static GetOnlineFeaturesRequest createOnlineFeatureRequest( - String projectName, String featureName, String entityId, int entityValue) { - return GetOnlineFeaturesRequest.newBuilder() + String featureTableName, + String featureName, + String entityId, + int entityValue) { + return GetOnlineFeaturesRequestV2.newBuilder() .setProject(projectName) - .addFeatures(FeatureReference.newBuilder().setName(featureName).build()) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable(featureTableName) + .setName(featureName) + .build()) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields(entityId, Value.newBuilder().setInt64Val(entityValue).build())) .build(); } - public static void applyFeatureSet( - CoreSimpleAPIClient secureApiClient, - String projectName, - String entityId, - String featureName) { - List> entities = new ArrayList<>(); - entities.add(Pair.of(entityId, ValueProto.ValueType.Enum.INT64)); - List> features = new ArrayList<>(); - features.add(Pair.of(featureName, ValueProto.ValueType.Enum.INT64)); - String featureSetName = "test_1"; - FeatureSetProto.FeatureSet expectedFeatureSet = - AuthTestUtils.createFeatureSet( - AuthTestUtils.getDefaultSource(), projectName, featureSetName, entities, features); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - waitAtMost(2, TimeUnit.MINUTES) - .until( - () -> { - return secureApiClient.simpleGetFeatureSet(projectName, featureSetName).getMeta(); - }, - hasProperty("status", equalTo(FeatureSetStatus.STATUS_READY))); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet(projectName, featureSetName); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); - assertEquals(FeatureSetStatus.STATUS_READY, actualFeatureSet.getMeta().getStatus()); - } - public static CoreSimpleAPIClient getSecureApiClientForCore( int feastCorePort, Map options) { CallCredentials callCredentials = null; diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 98836f9156..6cc4529bdb 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -27,6 +27,7 @@ @SpringBootTest public class BaseAuthIT { + static final String FEATURE_TABLE_NAME = "featuretable_1"; static final String FEATURE_NAME = "feature_1"; static final String ENTITY_ID = "entity_id"; static final String PROJECT_NAME = "project_1"; @@ -42,8 +43,6 @@ public class BaseAuthIT { static final String CORE = "core_1"; - static final String JOB_CONTROLLER = "jobcontroller_1"; - static final String HYDRA = "hydra_1"; static final int HYDRA_PORT = 4445; diff --git a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java b/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java index 3886bfbb8e..d526bbee75 100644 --- a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java +++ b/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java @@ -43,8 +43,12 @@ public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String .getFeatureSet(); } - public void simpleApplyEntity(EntityProto.EntitySpecV2 entitySpec) { - stub.applyEntity(CoreServiceProto.ApplyEntityRequest.newBuilder().setSpec(entitySpec).build()); + public void simpleApplyEntity(String projectName, EntityProto.EntitySpecV2 entitySpec) { + stub.applyEntity( + CoreServiceProto.ApplyEntityRequest.newBuilder() + .setProject(projectName) + .setSpec(entitySpec) + .build()); } public EntityProto.Entity getEntity(String projectName, String name) { @@ -56,9 +60,13 @@ public EntityProto.Entity getEntity(String projectName, String name) { .getEntity(); } - public void simpleApplyFeatureTable(FeatureTableProto.FeatureTableSpec featureTable) { + public void simpleApplyFeatureTable( + String projectName, FeatureTableProto.FeatureTableSpec featureTable) { stub.applyFeatureTable( - CoreServiceProto.ApplyFeatureTableRequest.newBuilder().setTableSpec(featureTable).build()); + CoreServiceProto.ApplyFeatureTableRequest.newBuilder() + .setProject(projectName) + .setTableSpec(featureTable) + .build()); } public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java index 476b017c6e..8f2440d247 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java @@ -20,21 +20,28 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; +import com.google.common.collect.ImmutableMap; import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Request; import com.squareup.okhttp.Response; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.common.it.DataGenerator; +import feast.proto.core.EntityProto; +import feast.proto.core.FeatureTableProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; +import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; import io.grpc.ManagedChannel; import java.io.File; import java.io.IOException; import java.time.Duration; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import org.junit.ClassRule; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.runners.model.InitializationError; import org.springframework.boot.test.context.SpringBootTest; @@ -58,6 +65,8 @@ @Testcontainers public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { + CoreSimpleAPIClient coreClient; + FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; static final Map options = new HashMap<>(); static final int FEAST_SERVING_PORT = 6566; @@ -72,11 +81,6 @@ public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { .withExposedService( CORE, FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService( - JOB_CONTROLLER, - FEAST_JOB_CONTROLLER_PORT, Wait.forLogMessage(".*gRPC Server started.*\\n", 1) .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); @@ -96,6 +100,35 @@ static void globalSetup() throws IOException, InitializationError, InterruptedEx options.put("grant_type", GRANT_TYPE); } + @BeforeEach + public void initState() { + coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); + EntityProto.EntitySpecV2 entitySpec = + DataGenerator.createEntitySpecV2( + ENTITY_ID, + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); + + expectedFeatureTableSpec = + DataGenerator.createFeatureTableSpec( + FEATURE_TABLE_NAME, + Arrays.asList(ENTITY_ID), + new HashMap<>() { + { + put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); + } + }, + 7200, + ImmutableMap.of("feat_key2", "feat_value2")) + .toBuilder() + .setBatchSource( + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .build(); + coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); + } + /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ @Test public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { @@ -111,37 +144,47 @@ public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOExcepti @Test public void shouldAllowUnauthenticatedGetOnlineFeatures() { - // apply feature set - CoreSimpleAPIClient coreClient = - AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); + FeatureTableProto.FeatureTable actualFeatureTable = + coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); + assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); + assertEquals( + expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); + ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); + assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @Test void canGetOnlineFeaturesIfAuthenticated() { - // apply feature set - CoreSimpleAPIClient coreClient = - AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); + FeatureTableProto.FeatureTable actualFeatureTable = + coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); + assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); + assertEquals( + expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); + ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } } diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java index 50111fcd15..e5a4d7ad1f 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; import feast.proto.types.ValueProto.Value; @@ -34,7 +34,6 @@ import java.util.Map; import org.junit.ClassRule; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.runners.model.InitializationError; import org.springframework.boot.test.context.SpringBootTest; @@ -80,11 +79,6 @@ public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { FEAST_CORE_PORT, Wait.forLogMessage(".*gRPC Server started.*\\n", 1) .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService( - JOB_CONTROLLER, - FEAST_JOB_CONTROLLER_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); @@ -134,24 +128,19 @@ static void globalSetup() throws IOException, InitializationError, InterruptedEx coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); } - @BeforeEach - public void setUp() { - // seed core - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); - } - @Test public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); Exception exception = assertThrows( StatusRuntimeException.class, () -> { - servingStub.getOnlineFeatures(onlineFeatureRequest); + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); }); String expectedMessage = "UNAUTHENTICATED: Authentication failed"; @@ -165,13 +154,15 @@ void canGetOnlineFeaturesIfAdmin() { // apply feature set ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @@ -182,13 +173,15 @@ void canGetOnlineFeaturesIfProjectMember() { memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @@ -199,12 +192,13 @@ void cantGetOnlineFeaturesIfNotProjectMember() { notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); StatusRuntimeException exception = assertThrows( StatusRuntimeException.class, - () -> servingStub.getOnlineFeatures(onlineFeatureRequest)); + () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); String expectedMessage = String.format( diff --git a/serving/src/test/java/feast/serving/it/TestUtils.java b/serving/src/test/java/feast/serving/it/TestUtils.java index 0c72df1ea8..6772dade9c 100644 --- a/serving/src/test/java/feast/serving/it/TestUtils.java +++ b/serving/src/test/java/feast/serving/it/TestUtils.java @@ -90,7 +90,7 @@ public static void applyFeatureTable( .setBatchSource( DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) .build(); - secureApiClient.simpleApplyFeatureTable(expectedFeatureTableSpec); + secureApiClient.simpleApplyFeatureTable(projectName, expectedFeatureTableSpec); FeatureTable actualFeatureTable = secureApiClient.simpleGetFeatureTable(projectName, featureTableName); assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); @@ -98,7 +98,7 @@ public static void applyFeatureTable( public static void applyEntity( CoreSimpleAPIClient coreApiClient, String projectName, EntitySpecV2 entitySpec) { - coreApiClient.simpleApplyEntity(entitySpec); + coreApiClient.simpleApplyEntity(projectName, entitySpec); String entityName = entitySpec.getName(); Entity actualEntity = coreApiClient.getEntity(projectName, entityName); assertEquals(entitySpec.getName(), actualEntity.getSpec().getName()); diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index b41a75918f..2775f76315 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -17,7 +17,6 @@ package feast.serving.service; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -27,28 +26,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListProjectsRequest; import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.FeatureSetProto; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; import feast.proto.core.FeatureTableProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.core.StoreProto.Store; -import feast.proto.core.StoreProto.Store.Subscription; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.types.ValueProto; -import feast.serving.exception.SpecRetrievalException; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; -import feast.storage.api.retriever.FeatureSetRequest; -import java.util.HashMap; -import java.util.List; import java.util.Map; import org.junit.Before; import org.junit.Rule; @@ -78,25 +67,6 @@ public void setUp() { initMocks(this); this.store = Store.newBuilder().build(); - this.featureSetSpecs = new HashMap<>(); - - this.setupFeatureSetAndStoreSubscription( - "project", - "fs1", - List.of( - FeatureSpec.newBuilder().setName("feature").build(), - FeatureSpec.newBuilder().setName("feature2").build())); - - this.setupFeatureSetAndStoreSubscription( - "default", - "fs2", - List.of( - FeatureSpec.newBuilder().setName("feature3").build(), - FeatureSpec.newBuilder().setName("feature4").build(), - FeatureSpec.newBuilder().setName("feature5").build())); - - this.setupFeatureSetAndStoreSubscription( - "default", "fs3", List.of(FeatureSpec.newBuilder().setName("feature4").build())); this.setupProject("default"); this.featureTableEntities = ImmutableList.of("entity1"); @@ -137,9 +107,6 @@ private void setupProject(String project) { } private void setupFeatureTableAndProject(String project) { - ImmutableMap featureTable1Features = - this.featureTable1Features; - FeatureTableProto.FeatureTable featureTable1 = FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable1Spec).build(); FeatureTableProto.FeatureTable featureTable2 = @@ -156,37 +123,6 @@ private void setupFeatureTableAndProject(String project) { .build()); } - private void setupFeatureSetAndStoreSubscription( - String project, String name, List featureSpecs) { - FeatureSetSpec fsSpec = - FeatureSetSpec.newBuilder() - .setProject(project) - .setName(name) - .addAllFeatures(featureSpecs) - .build(); - this.featureSetSpecs.put(String.format("%s", name), fsSpec); - - this.store = - this.store - .toBuilder() - .addSubscriptions(Subscription.newBuilder().setProject(project).setName(name).build()) - .build(); - - // collect the different versions the featureset with the given name - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(fsSpec).build(); - - when(coreService.listFeatureSets( - ListFeatureSetsRequest.newBuilder() - .setFilter( - ListFeatureSetsRequest.Filter.newBuilder() - .setProject(project) - .setFeatureSetName(name) - .build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet).build()); - } - @Test public void shouldRegisterStoreWithCore() { verify(coreService, times(1)).registerStore(cachedSpecService.getStore()); @@ -229,132 +165,4 @@ public void shouldPopulateAndReturnDifferentFeatureTables() { cachedSpecService.getFeatureTableSpec("default", featureReference3), equalTo(this.featureTable2Spec)); } - - @Test - public void shouldPopulateAndReturnFeatureSets() { - // test that CachedSpecService can retrieve fully qualified feature references. - cachedSpecService.populateCache(); - FeatureReference fs1fr1 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setFeatureSet("fs1") - .build(); - FeatureReference fs1fr2 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature2") - .setFeatureSet("fs1") - .build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs1fr2)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .addFeatureReference(fs1fr2) - .setSpec(featureSetSpecs.get("fs1")) - .build()))); - } - - @Test - public void shouldPopulateAndReturnFeatureSetWithDefaultProjectIfProjectNotSupplied() { - // test that CachedSpecService will use default project when project unspecified - FeatureReference fs2fr3 = - FeatureReference.newBuilder().setName("feature3").setFeatureSet("fs2").build(); - // check that this is true for references in where feature set is unspecified - FeatureReference fs2fr5 = FeatureReference.newBuilder().setName("feature5").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs2fr3, fs2fr5)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .addFeatureReference(fs2fr5) - .setSpec(featureSetSpecs.get("fs2")) - .build()))); - } - - @Test - public void shouldPopulateAndReturnClosestFeatureSetIfFeatureSetNotSupplied() { - // test that CachedSpecService will try to match a featureset without a featureset name in - // reference - FeatureReference fs1fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - - // check that this is true for reference in which project is unspecified - FeatureReference fs2fr3 = FeatureReference.newBuilder().setName("feature3").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs2fr3)), - containsInAnyOrder( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .setSpec(featureSetSpecs.get("fs1")) - .build(), - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .setSpec(featureSetSpecs.get("fs2")) - .build()) - .toArray())); - } - - @Test - public void shouldPopulateAndReturnFeatureSetsGivenFeaturesFromDifferentFeatureSets() { - cachedSpecService.populateCache(); - FeatureReference fs1fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - - FeatureReference fs2fr3 = - FeatureReference.newBuilder().setProject("default").setName("feature3").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs2fr3)), - containsInAnyOrder( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .setSpec(featureSetSpecs.get("fs1")) - .build(), - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .setSpec(featureSetSpecs.get("fs2")) - .build()) - .toArray())); - } - - @Test - public void shouldPopulateAndReturnFeatureSetGivenFeaturesFromSameFeatureSet() { - cachedSpecService.populateCache(); - FeatureReference fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - FeatureReference fr2 = - FeatureReference.newBuilder().setProject("project").setName("feature2").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fr1, fr2)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fr1) - .addFeatureReference(fr2) - .setSpec(featureSetSpecs.get("fs1")) - .build()))); - } - - @Test - public void shouldThrowExceptionWhenMultipleFeatureSetMapToFeatureReference() - throws SpecRetrievalException { - // both fs2 and fs3 have the feature with the same name. - // using a generic feature reference only specifying the feature's name - // should cause a multiple feature sets to match and throw an error - FeatureReference fs2fr4 = FeatureReference.newBuilder().setName("feature4").build(); - FeatureReference fs3fr4 = FeatureReference.newBuilder().setName("feature4").build(); - - expectedException.expect(SpecRetrievalException.class); - cachedSpecService.getFeatureSets(List.of(fs2fr4, fs3fr4)); - } } diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index f2a8038a97..e98f4204ee 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -16,32 +16,33 @@ */ package feast.serving.service; +import static feast.common.it.DataGenerator.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.Lists; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureTableProto.FeatureTableSpec; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.FieldProto; -import feast.proto.types.ValueProto.Value; +import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.FeatureSetRequest; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.redis.retriever.OnlineRetriever; import feast.storage.connectors.redis.retriever.RedisOnlineRetriever; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.Before; @@ -57,15 +58,20 @@ public class OnlineServingServiceTest { @Mock Tracer tracer; @Mock RedisOnlineRetriever retriever; + @Mock OnlineRetriever retrieverV2; private OnlineServingService onlineServingService; + private OnlineServingServiceV2 onlineServingServiceV2; List testFeatureRows; + List mockedFeatureRows; + List featureSpecs; @Before public void setUp() { initMocks(this); onlineServingService = new OnlineServingService(retriever, specService, tracer); + onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); // create fake feature rows for testing. testFeatureRows = new ArrayList<>(); @@ -74,15 +80,21 @@ public void setUp() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - FieldProto.Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(createInt64Value(1)) + .build(), FieldProto.Field.newBuilder() .setName("entity2") - .setValue(strValue("a")) + .setValue(createStrValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(createInt64Value(1)) .build(), - FieldProto.Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), FieldProto.Field.newBuilder() .setName("feature2") - .setValue(intValue(1)) + .setValue(createInt64Value(1)) .build())) .setFeatureSet("featureSet") .build()); @@ -92,15 +104,21 @@ public void setUp() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - FieldProto.Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(createInt64Value(2)) + .build(), FieldProto.Field.newBuilder() .setName("entity2") - .setValue(strValue("b")) + .setValue(createStrValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(createInt64Value(2)) .build(), - FieldProto.Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), FieldProto.Field.newBuilder() .setName("feature2") - .setValue(intValue(2)) + .setValue(createInt64Value(2)) .build())) .setFeatureSet("featureSet") .build()); @@ -111,306 +129,338 @@ public void setUp() { .toBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(50)) .build()); - } - - @Test - public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) - .build(); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); - - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + mockedFeatureRows = new ArrayList<>(); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(2)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .setFeatureValue(createStrValue("1")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build()) - .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); + .setFeatureValue(createStrValue("2")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .setFeatureValue(createStrValue("3")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build()) + .setFeatureValue(createStrValue("4")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_3") + .build()) + .setFeatureValue(createStrValue("5")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .setFeatureValue(createStrValue("6")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(50).build()) + .build()); + + featureSpecs = new ArrayList<>(); + featureSpecs.add( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()); + featureSpecs.add( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()); } @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { - // some keys not present, should have empty values - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) + public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build(); + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build(); + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); + + List> entityKeyList1 = new ArrayList<>(); + List> entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(0))); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(1))); + entityKeyList2.add(Optional.of(mockedFeatureRows.get(2))); + entityKeyList2.add(Optional.of(mockedFeatureRows.get(3))); + + List>> featureRows = List.of(entityKeyList1, entityKeyList2); + + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.empty()); - - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createStrValue("1")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", Value.newBuilder().build()) - .putStatuses("feature1", FieldStatus.NOT_FOUND) - .putFields("project/feature2", Value.newBuilder().build()) - .putStatuses("project/feature2", FieldStatus.NOT_FOUND) + .putFields("featuretable_1:feature_1", createStrValue("3")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("4")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { - // keys present, but considered stale when compared to maxAge - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(2))); - - FeatureSetSpec spec = - getFeatureSetSpec().toBuilder().setMaxAge(Duration.newBuilder().setSeconds(1)).build(); - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(spec) + public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build(); + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build(); + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); + List> entityKeyList1 = new ArrayList<>(); + List> entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(0))); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(1))); + entityKeyList2.add(Optional.of(mockedFeatureRows.get(4))); - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", Value.newBuilder().build()) - .putStatuses("feature1", FieldStatus.OUTSIDE_MAX_AGE) - .putFields("project/feature2", Value.newBuilder().build()) - .putStatuses("project/feature2", FieldStatus.OUTSIDE_MAX_AGE) - .build()) - .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } + List>> featureRows = List.of(entityKeyList1, entityKeyList2); - @Test - public void shouldFilterOutUndesiredRows() { - // requested rows less than the rows available in the featureset - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of(FeatureReference.newBuilder().setName("feature1").build())); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) - .build(); + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createStrValue("1")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.NOT_FOUND) + .putFields("featuretable_1:feature_2", createEmptyValue()) + .putStatuses("featuretable_1:feature_2", FieldStatus.NOT_FOUND) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @Test - public void shouldApplyProjectOverrideInRequest() { - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder() - .setName("feature2") - .setProject("project") - .build())) - .toBuilder() - .setProject("project") + public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build(); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build(); + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); + + List> entityKeyList1 = new ArrayList<>(); + List> entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(5))); + entityKeyList1.add(Optional.of(mockedFeatureRows.get(1))); + entityKeyList2.add(Optional.of(mockedFeatureRows.get(5))); + entityKeyList2.add(Optional.of(mockedFeatureRows.get(1))); + + List>> featureRows = List.of(entityKeyList1, entityKeyList2); + + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())) + .thenReturn( + FeatureTableSpec.newBuilder() + .setName("featuretable_1") + .addEntities("entity1") + .addEntities("entity2") + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .setMaxAge(Duration.newBuilder().setSeconds(1)) + .build()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSets(request.getFeaturesList(), "project")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(2)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } - private Value intValue(int val) { - return Value.newBuilder().setInt32Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - - private FeatureSetSpec getFeatureSetSpec() { - return FeatureSetSpec.newBuilder() - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) + private FeatureTableSpec getFeatureTableSpec() { + return FeatureTableSpec.newBuilder() + .setName("featuretable_1") + .addEntities("entity1") + .addEntities("entity2") + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .setMaxAge(Duration.newBuilder().setSeconds(120)) .build(); } - private GetOnlineFeaturesRequest getOnlineFeaturesRequest( - List featureReferences) { - return GetOnlineFeaturesRequest.newBuilder() - .setOmitEntitiesInResponse(false) + private GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2( + String projectName, List featureReferences) { + return GetOnlineFeaturesRequestV2.newBuilder() + .setProject(projectName) .addAllFeatures(featureReferences) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a"))) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", createInt64Value(1)) + .putFields("entity2", createStrValue("a"))) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b"))) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", createInt64Value(2)) + .putFields("entity2", createStrValue("b"))) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build()) .build(); } } diff --git a/serving/src/test/resources/docker-compose/docker-compose-it.yml b/serving/src/test/resources/docker-compose/docker-compose-it.yml index edef066fd5..fb7fb1f6ad 100644 --- a/serving/src/test/resources/docker-compose/docker-compose-it.yml +++ b/serving/src/test/resources/docker-compose/docker-compose-it.yml @@ -18,20 +18,6 @@ services: - -jar - /opt/feast/feast-core.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - jobcontroller: - image: gcr.io/kf-feast/feast-jobcontroller:latest - volumes: - - ./job-controller/application-it.yml:/etc/feast/application.yml - depends_on: - - kafka - ports: - - 6570:6570 - command: - - java - - -jar - - /opt/feast/feast-job-controller.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml kafka: image: confluentinc/cp-kafka:5.2.1 diff --git a/serving/src/test/resources/docker-compose/job-controller/application-it.yml b/serving/src/test/resources/docker-compose/job-controller/application-it.yml deleted file mode 100644 index 118ce37872..0000000000 --- a/serving/src/test/resources/docker-compose/job-controller/application-it.yml +++ /dev/null @@ -1,16 +0,0 @@ -feast: - core-host: core - jobs: - enabled: true - polling_interval_milliseconds: 30000 - job_update_timeout_seconds: 240 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: {} - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" \ No newline at end of file diff --git a/storage/api/src/main/java/feast/storage/common/retry/Retriable.java b/storage/api/src/main/java/feast/storage/common/retry/Retriable.java deleted file mode 100644 index 2c92c85175..0000000000 --- a/storage/api/src/main/java/feast/storage/common/retry/Retriable.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.common.retry; - -public interface Retriable { - void execute() throws Exception; - - Boolean isExceptionRetriable(Exception e); - - void cleanUpAfterFailure(); -} diff --git a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java b/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java deleted file mode 100644 index 773abd57d6..0000000000 --- a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.common.testing; - -import com.google.common.hash.Hashing; -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FeatureRowProto.FeatureRow.Builder; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.*; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.commons.lang3.RandomStringUtils; - -@SuppressWarnings("WeakerAccess") -public class TestUtil { - - /** - * Create a Feature Row with random value according to the FeatureSetSpec - * - * @param featureSet {@link FeatureSet} - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet) { - ThreadLocalRandom random = ThreadLocalRandom.current(); - int randomStringSizeMaxSize = 12; - return createRandomFeatureRow(featureSet, random.nextInt(0, randomStringSizeMaxSize) + 4); - } - - /** - * Create a Feature Row with random value according to the FeatureSet. - * - *

The Feature Row created contains fields according to the entities and features defined in - * FeatureSet, matching the value type of the field, with randomized value for testing. - * - * @param featureSet {@link FeatureSet} - * @param randomStringSize number of characters for the generated random string - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet, int randomStringSize) { - - Instant time = Instant.now(); - Timestamp timestamp = - Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build(); - - Builder builder = - FeatureRow.newBuilder() - .setFeatureSet(getFeatureSetReference(featureSet)) - .setEventTimestamp(timestamp); - - featureSet - .getSpec() - .getEntitiesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - featureSet - .getSpec() - .getFeaturesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - return builder.build(); - } - - private static String getFeatureSetReference(FeatureSet featureSet) { - FeatureSetSpec spec = featureSet.getSpec(); - return String.format("%s/%s:%d", spec.getProject(), spec.getName()); - } - - /** - * Create a random Feast {@link Value} of {@link ValueType.Enum}. - * - * @param type {@link ValueType.Enum} - * @param randomStringSize number of characters for the generated random string - * @return {@link Value} - */ - public static Value createRandomValue(ValueType.Enum type, int randomStringSize) { - Value.Builder builder = Value.newBuilder(); - ThreadLocalRandom random = ThreadLocalRandom.current(); - - switch (type) { - case INVALID: - case UNRECOGNIZED: - throw new IllegalArgumentException("Invalid ValueType: " + type); - case BYTES: - builder.setBytesVal( - ByteString.copyFrom(RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())); - break; - case STRING: - builder.setStringVal(RandomStringUtils.randomAlphanumeric(randomStringSize)); - break; - case INT32: - builder.setInt32Val(random.nextInt()); - break; - case INT64: - builder.setInt64Val(random.nextLong()); - break; - case DOUBLE: - builder.setDoubleVal(random.nextDouble()); - break; - case FLOAT: - builder.setFloatVal(random.nextFloat()); - break; - case BOOL: - builder.setBoolVal(random.nextBoolean()); - break; - case BYTES_LIST: - builder.setBytesListVal( - BytesList.newBuilder() - .addVal( - ByteString.copyFrom( - RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())) - .build()); - break; - case STRING_LIST: - builder.setStringListVal( - StringList.newBuilder() - .addVal(RandomStringUtils.randomAlphanumeric(randomStringSize)) - .build()); - break; - case INT32_LIST: - builder.setInt32ListVal(Int32List.newBuilder().addVal(random.nextInt()).build()); - break; - case INT64_LIST: - builder.setInt64ListVal(Int64List.newBuilder().addVal(random.nextLong()).build()); - break; - case DOUBLE_LIST: - builder.setDoubleListVal(DoubleList.newBuilder().addVal(random.nextDouble()).build()); - break; - case FLOAT_LIST: - builder.setFloatListVal(FloatList.newBuilder().addVal(random.nextFloat()).build()); - break; - case BOOL_LIST: - builder.setBoolListVal(BoolList.newBuilder().addVal(random.nextBoolean()).build()); - break; - } - return builder.build(); - } - - /** - * Create a field object with given name and type. - * - * @param name of the field. - * @param value of the field. Should be compatible with the valuetype given. - * @param valueType type of the field. - * @return Field object - */ - public static Field field(String name, Object value, ValueType.Enum valueType) { - Field.Builder fieldBuilder = Field.newBuilder().setName(name); - switch (valueType) { - case INT32: - return fieldBuilder.setValue(Value.newBuilder().setInt32Val((int) value)).build(); - case INT64: - return fieldBuilder.setValue(Value.newBuilder().setInt64Val((int) value)).build(); - case FLOAT: - return fieldBuilder.setValue(Value.newBuilder().setFloatVal((float) value)).build(); - case DOUBLE: - return fieldBuilder.setValue(Value.newBuilder().setDoubleVal((double) value)).build(); - case STRING: - return fieldBuilder.setValue(Value.newBuilder().setStringVal((String) value)).build(); - default: - throw new IllegalStateException("Unexpected valueType: " + value.getClass()); - } - } - - public static String hash(String input) { - return Hashing.murmur3_32().hashString(input, StandardCharsets.UTF_8).toString(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java deleted file mode 100644 index c843d31127..0000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.junit.Assert.*; - -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.types.FeatureRowProto; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.ValueType; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import org.junit.Test; - -public class FeatureRowDecoderTest { - - private FeatureSetProto.EntitySpec entity = - FeatureSetProto.EntitySpec.newBuilder().setName("entity1").build(); - - private FeatureSetSpec spec = - FeatureSetSpec.newBuilder() - .addAllEntities(Collections.singletonList(entity)) - .addFeatures( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueType.Enum.FLOAT)) - .addFeatures( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueType.Enum.INT32)) - .setName("feature_set_name") - .build(); - - @Test - public void shouldDecodeValidEncodedFeatureRowV2() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature2", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - @Test - public void shouldDecodeValidFeatureRowV2WithIncompleteFields() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .build(); - - // should decode missing fields as fields with unset value. - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields(Field.newBuilder().setName("feature2").setValue(Value.newBuilder().build())) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - @Test - public void shouldDecodeValidFeatureRowV2AndIgnoreExtraFields() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature2", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature3", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setStringVal("data"))) - .build(); - - // should decode missing fields as fields with unset value. - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - // TODO: remove this test in Feast 0.7 when support for Feature Row v1 encoding is removed - @Test - public void shouldDecodeValidEncodedFeatureRowV1() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt32Val(2))) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java deleted file mode 100644 index 419ce8e0a9..0000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializer; -import feast.storage.connectors.redis.serializer.RedisKeySerializer; -import io.lettuce.core.KeyValue; -import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; -import java.util.List; -import java.util.Optional; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; - -public class RedisClusterOnlineRetrieverTest { - - @Mock StatefulRedisClusterConnection connection; - - @Mock RedisAdvancedClusterCommands syncCommands; - - RedisKeySerializer serializer = new RedisKeyPrefixSerializer("test:"); - - RedisKeySerializer fallbackSerializer = new RedisKeyPrefixSerializer(""); - - private List redisKeys; - private FeatureSetRequest featureSetRequest; - private List entityRows; - private List featureRows; - - @Before - public void setUp() { - initMocks(this); - when(connection.sync()).thenReturn(syncCommands); - redisKeys = - Lists.newArrayList( - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) - .build(), - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build())) - .build()); - - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .addFeatures(FeatureSpec.newBuilder().setName("feature1")) - .addFeatures(FeatureSpec.newBuilder().setName("feature2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) // default - .build(); - - featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(featureSetSpec) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - - entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(2)).build(), - Field.newBuilder().setValue(intValue(2)).build())) - .build()); - } - - @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = - KeyValue.from(serializedKey2, Optional.of(featureRows.get(1).toByteArray())); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer) - .withFallbackSerializer(fallbackSerializer) - .build(); - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - - // check that fallback is used only when there's something to fallback - verify(syncCommands, never()).mget(); - } - - @Test - public void shouldReturnNullIfKeysNotPresent() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = KeyValue.empty(serializedKey2); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer).build(); - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.empty()); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldUseFallbackIfAvailable() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - byte[] fallbackSerializedKey2 = fallbackSerializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = KeyValue.empty(serializedKey2); - KeyValue fallbackKeyValue2 = - KeyValue.from(serializedKey2, Optional.of(featureRows.get(1).toByteArray())); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - List> fallbackFeatureRowBytes = Lists.newArrayList(fallbackKeyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer) - .withFallbackSerializer(fallbackSerializer) - .build(); - - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - when(syncCommands.mget(fallbackSerializedKey2)).thenReturn(fallbackFeatureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java deleted file mode 100644 index 1292f4ab0d..0000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.AbstractMessageLite; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import io.lettuce.core.KeyValue; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; - -public class RedisOnlineRetrieverTest { - - @Mock StatefulRedisConnection connection; - - @Mock RedisCommands syncCommands; - - private OnlineRetriever redisOnlineRetriever; - private byte[][] redisKeyList; - - @Before - public void setUp() { - initMocks(this); - when(connection.sync()).thenReturn(syncCommands); - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - redisKeyList = - Lists.newArrayList( - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) - .build(), - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build())) - .build()) - .stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - } - - @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(getFeatureSetSpec()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - List entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(2)).build(), - Field.newBuilder().setValue(intValue(2)).build())) - .build()); - - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); - - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldReturnNullIfKeysNotPresent() { - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(getFeatureSetSpec()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - List entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build()); - - List> featureRowBytes = - featureRows.stream() - .map(row -> KeyValue.from(new byte[1], Optional.of(row.toByteArray()))) - .collect(Collectors.toList()); - featureRowBytes.add(null); - - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.empty()); - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - - private FeatureSetSpec getFeatureSetSpec() { - return FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .addFeatures(FeatureSpec.newBuilder().setName("feature1")) - .addFeatures(FeatureSpec.newBuilder().setName("feature2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) // default - .build(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java index c73108e118..e663cf81ee 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java @@ -18,35 +18,28 @@ import static org.junit.Assert.*; -import com.google.common.collect.Lists; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FieldProto; +import feast.proto.storage.RedisProto.RedisKeyV2; import feast.proto.types.ValueProto; import org.junit.Test; public class RedisKeyPrefixSerializerTest { - private RedisKey key = - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(ValueProto.Value.newBuilder().setInt64Val(1)) - .build())) + private RedisKeyV2 key = + RedisKeyV2.newBuilder() + .addEntityNames("entity1") + .addEntityValues(ValueProto.Value.newBuilder().setInt64Val(1)) .build(); @Test public void shouldPrependKey() { - RedisKeyPrefixSerializer serializer = new RedisKeyPrefixSerializer("namespace:"); + RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2("namespace:"); String keyWithPrefix = new String(serializer.serialize(key)); assertEquals(String.format("namespace:%s", new String(key.toByteArray())), keyWithPrefix); } @Test public void shouldNotPrependKeyIfEmptyString() { - RedisKeyPrefixSerializer serializer = new RedisKeyPrefixSerializer(""); + RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2(""); assertArrayEquals(key.toByteArray(), serializer.serialize(key)); } }