From d743cc790b2be87432b3231eea4ee36f979be8a4 Mon Sep 17 00:00:00 2001 From: Neelan Date: Tue, 19 Jun 2018 11:04:03 -0500 Subject: [PATCH] add storage strategy to dim config object (#717) --- CHANGELOG.md | 4 ++ .../DefaultKeyValueStoreDimensionConfig.java | 8 ++++ .../config/dimension/DimensionConfig.java | 8 ++++ .../impl/KeyValueStoreDimension.java | 2 +- .../TypeAwareDimensionLoadTaskSpec.groovy | 14 +++++++ .../DimensionsServletComponentSpec.groovy | 2 +- .../ExpectedTablesFullViewEndpointSpec.groovy | 8 ++-- .../config/dimension/TestDimensionConfig.java | 38 +++++++++++++++++++ .../data/config/dimension/TestDimensions.java | 22 ++++++++++- 9 files changed, 99 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb5a046029..bb54f7e694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ pull request if there was one. ### Added: +- [Add storageStrategy as a field of the DimensionConfig class](https://github.com/yahoo/fili/issues/718) + * Adds getStorageStrategy as a field of the dimensionConfig class. + * Passes the storage strategy to the KeyValueStoreDimension Constructor + - [Add more tests to RegisteredLookupMetadataLoadTask](https://github.com/yahoo/fili/pull/673) * Adds tests to make sure the load tasks can update status correctly. diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DefaultKeyValueStoreDimensionConfig.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DefaultKeyValueStoreDimensionConfig.java index 9aa9f5c6c7..193c482d06 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DefaultKeyValueStoreDimensionConfig.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DefaultKeyValueStoreDimensionConfig.java @@ -6,6 +6,7 @@ import com.yahoo.bard.webservice.data.dimension.DimensionField; import com.yahoo.bard.webservice.data.dimension.KeyValueStore; import com.yahoo.bard.webservice.data.dimension.SearchProvider; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import java.util.LinkedHashSet; import javax.validation.constraints.NotNull; @@ -24,6 +25,7 @@ public class DefaultKeyValueStoreDimensionConfig implements DimensionConfig { private final LinkedHashSet defaultDimensionFields; private final KeyValueStore keyValueStore; private final SearchProvider searchProvider; + private final StorageStrategy storageStrategy; /** * Construct a DefaultKeyValueStoreDimensionConfig instance from dimension name, dimension fields and @@ -59,6 +61,7 @@ public DefaultKeyValueStoreDimensionConfig( this.defaultDimensionFields = defaultDimensionFields; this.keyValueStore = keyValueStore; this.searchProvider = searchProvider; + this.storageStrategy = StorageStrategy.LOADED; } /** @@ -141,4 +144,9 @@ public KeyValueStore getKeyValueStore() { public SearchProvider getSearchProvider() { return searchProvider; } + + @Override + public StorageStrategy getStorageStrategy() { + return storageStrategy; + } } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DimensionConfig.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DimensionConfig.java index e1b99708d1..9226e88280 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DimensionConfig.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DimensionConfig.java @@ -6,6 +6,7 @@ import com.yahoo.bard.webservice.data.dimension.KeyValueStore; import com.yahoo.bard.webservice.data.dimension.SearchProvider; import com.yahoo.bard.webservice.data.dimension.impl.KeyValueStoreDimension; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import java.util.LinkedHashSet; @@ -50,6 +51,13 @@ public interface DimensionConfig { */ String getDescription(); + /** + * The storage strategy for this dimension. + * + * @return The storage strategy of the dimension + */ + StorageStrategy getStorageStrategy(); + /** * The set of fields for this dimension. * diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java index 90603e3c1d..21786c4ba9 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java @@ -312,7 +312,7 @@ public KeyValueStoreDimension(DimensionConfig dimensionConfig) { dimensionConfig.getSearchProvider(), dimensionConfig.getDefaultDimensionFields(), dimensionConfig.isAggregatable(), - StorageStrategy.LOADED + dimensionConfig.getStorageStrategy() ); } diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoadTaskSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoadTaskSpec.groovy index 700b5588f1..2bf4f3e2d3 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoadTaskSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoadTaskSpec.groovy @@ -5,6 +5,7 @@ package com.yahoo.bard.webservice.data.config.dimension import com.yahoo.bard.webservice.data.dimension.DimensionDictionary import com.yahoo.bard.webservice.data.dimension.impl.KeyValueStoreDimension import com.yahoo.bard.webservice.data.dimension.impl.LookupDimension +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy import spock.lang.Specification /** @@ -40,6 +41,19 @@ class TypeAwareDimensionLoadTaskSpec extends Specification { then: dimensionDictionary.findByApiName("color").getClass() == KeyValueStoreDimension.class + + } + + def "Test dimension loader for dimension with storageStrategy none"() { + given: "A Type Aware Dimension LoadTask with a list of dimension configurations" + TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations) + + when: + typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary) + + then: + dimensionDictionary.findByApiName("color").getStorageStrategy() == StorageStrategy.NONE + dimensionConfigurations.find({it.apiName == 'color'}).getStorageStrategy() == StorageStrategy.NONE } def "Test dimension loader for a dimension type that is not defined"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy index 2b35ca8223..a722bd3a97 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy @@ -78,7 +78,7 @@ class DimensionsServletComponentSpec extends Specification { String expectedResponse = """{ "dimensions": [ - {"category": "General", "name": "color", "longName": "color", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/color", "cardinality": 0, "storageStrategy":"loaded"}, + {"category": "General", "name": "color", "longName": "color", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/color", "cardinality": 0, "storageStrategy":"none"}, {"category": "General", "name": "shape", "longName": "shape", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/shape", "cardinality": 38, "storageStrategy":"loaded"}, {"category": "General", "name": "size", "longName": "size", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/size", "cardinality": 0, "storageStrategy":"loaded"}, {"category": "General", "name": "model", "longName": "model", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/model", "cardinality": 21, "storageStrategy":"loaded"}, diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/ExpectedTablesFullViewEndpointSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/ExpectedTablesFullViewEndpointSpec.groovy index c21018a974..9944b92f62 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/ExpectedTablesFullViewEndpointSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/ExpectedTablesFullViewEndpointSpec.groovy @@ -967,7 +967,7 @@ class ExpectedTablesFullViewEndpointSpec extends BaseTableServletComponentSpec { "category": "General", "longName": "color", "name": "color", - "storageStrategy":"loaded", + "storageStrategy":"none", "fields": [ { "description":"Blue pigment", @@ -1176,7 +1176,7 @@ class ExpectedTablesFullViewEndpointSpec extends BaseTableServletComponentSpec { "category": "General", "longName": "color", "name": "color", - "storageStrategy":"loaded", + "storageStrategy":"none", "fields": [ { "description":"Blue pigment", @@ -1371,7 +1371,7 @@ class ExpectedTablesFullViewEndpointSpec extends BaseTableServletComponentSpec { "category": "General", "longName": "color", "name": "color", - "storageStrategy":"loaded", + "storageStrategy":"none", "fields": [ { "description":"Blue pigment", @@ -1580,7 +1580,7 @@ class ExpectedTablesFullViewEndpointSpec extends BaseTableServletComponentSpec { "category": "General", "longName": "color", "name": "color", - "storageStrategy":"loaded", + "storageStrategy":"none", "fields": [ { "description":"Blue pigment", diff --git a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java index b182fd9530..2a4030bbd0 100644 --- a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java +++ b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java @@ -7,6 +7,7 @@ import com.yahoo.bard.webservice.data.dimension.DimensionField; import com.yahoo.bard.webservice.data.dimension.KeyValueStore; import com.yahoo.bard.webservice.data.dimension.SearchProvider; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import java.util.LinkedHashSet; @@ -17,6 +18,7 @@ public class TestDimensionConfig implements DimensionConfig { private final TestApiDimensionName apiName; private final String physicalName; private final String description; + private final StorageStrategy storageStrategy; private LinkedHashSet fields; private LinkedHashSet defaultFields; @@ -48,6 +50,37 @@ public TestDimensionConfig( this.searchProvider = searchProvider; this.fields = fields; this.defaultFields = defaultFields; + this.storageStrategy = StorageStrategy.LOADED; + } + + /** + * Constructor. + * + * @param apiName API Name of the dimension + * @param physicalName Physical name of the dimension + * @param keyValueStore KeyValueStore for the dimension + * @param searchProvider SearchProvider for the dimension + * @param fields Fields of the dimension + * @param defaultFields Default fields of the dimension + * @param storageStrategy Storage Strategy of the dimension + */ + public TestDimensionConfig( + TestApiDimensionName apiName, + String physicalName, + KeyValueStore keyValueStore, + SearchProvider searchProvider, + LinkedHashSet fields, + LinkedHashSet defaultFields, + StorageStrategy storageStrategy + ) { + this.apiName = apiName; + this.physicalName = physicalName; + this.description = apiName.asName(); + this.keyValueStore = keyValueStore; + this.searchProvider = searchProvider; + this.fields = fields; + this.defaultFields = defaultFields; + this.storageStrategy = storageStrategy; } //CHECKSTYLE:OFF @@ -129,4 +162,9 @@ public SearchProvider getSearchProvider() { public String toString() { return "Dimension Config: apiName: " + apiName + " physicalName: " + physicalName + " fields: " + fields; } + + @Override + public StorageStrategy getStorageStrategy() { + return storageStrategy; + } } diff --git a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java index 4556f43825..cab1704d91 100644 --- a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java +++ b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java @@ -24,6 +24,7 @@ import com.yahoo.bard.webservice.data.dimension.impl.LuceneSearchProviderManager; import com.yahoo.bard.webservice.data.dimension.impl.NoOpSearchProviderManager; import com.yahoo.bard.webservice.data.dimension.impl.ScanSearchProvider; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import com.yahoo.bard.webservice.util.EnumUtils; import com.yahoo.bard.webservice.util.Utils; @@ -52,7 +53,7 @@ public TestDimensions() { dimensionNameConfigs = new LinkedHashMap<>(); dimensionNameConfigs.put(SIZE, TestDimensions.buildStandardDimensionConfig(SIZE)); dimensionNameConfigs.put(SHAPE, TestDimensions.buildStandardDimensionConfig(SHAPE)); - dimensionNameConfigs.put(COLOR, TestDimensions.buildStandardDimensionConfig(COLOR)); + dimensionNameConfigs.put(COLOR, TestDimensions.buildNonLoadedDimensionConfig(COLOR)); dimensionNameConfigs.put(OTHER, new TestDimensionConfig( OTHER, @@ -109,6 +110,25 @@ protected static TestDimensionConfig buildStandardDimensionConfig(TestApiDimensi ); } + /** + * Build a non-loaded dimension config. + * + * @param dimensionName Name of the dimension to build + * + * @return the standard dimension config + */ + protected static TestDimensionConfig buildNonLoadedDimensionConfig(TestApiDimensionName dimensionName) { + return new TestDimensionConfig( + dimensionName, + dimensionName.asName(), + getDefaultKeyValueStore(dimensionName), + getDefaultSearchProvider(), + getDefaultFields(), + getDefaultFields(), + StorageStrategy.NONE + ); + } + /** * Build a standard-config Lucene-backed dimension config. *