From 62ace2368584f68b8e38def43ff862fd31f72596 Mon Sep 17 00:00:00 2001 From: Jiaqi Liu <2257440489@qq.com> Date: Mon, 30 Oct 2017 10:43:15 -0700 Subject: [PATCH] Implement dimension metadata to indicate storage strategy --- CHANGELOG.md | 6 +- .../webservice/data/dimension/Dimension.java | 10 +++ .../impl/KeyValueStoreDimension.java | 48 ++++++++++-- .../dimension/metadata/StorageStrategy.java | 28 +++++++ .../web/endpoints/DimensionsServlet.java | 2 + .../DimensionToNameSerializerSpec.groovy | 78 ++++++++++--------- .../web/AggregatabilityValidationSpec.groovy | 4 +- .../DimensionsServletComponentSpec.groovy | 17 ++-- .../web/endpoints/TablesServletSpec.groovy | 1 + .../web/endpoints/TablesServletSpec.groovy | 1 + 10 files changed, 144 insertions(+), 51 deletions(-) create mode 100644 fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/metadata/StorageStrategy.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c3cb252b4..e52a9f20a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Current ### Added: +- [Implement dimension metadata to indicate storage strategy](https://github.com/yahoo/fili/pull/558) + * In order to allow clients to be notified if a dimension's values are browsable and searchable, a storage strategy + metadata is added to dimension. + - [Refactor ApiRequest](https://github.com/yahoo/fili/pull/538) * Add inteface layer to each type of API request class. The types of API request under the refactor are - `TablesApiRequest` @@ -17,7 +21,7 @@ Current - `SlicesApiRequest` - `MetricsApiRequest` - `JobsApiRequest` - + - [Implement Query Split Logging](https://github.com/yahoo/fili/pull/537) * Include metrics in logging to allow for better evaluation of the impact of caching for split queries. - Currently there is only a binary flag (`BardQueryInfo.cached`) that is inconsistently set for split queries diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/Dimension.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/Dimension.java index 4bcaf9818f..4fcd05885f 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/Dimension.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/Dimension.java @@ -2,6 +2,7 @@ // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.data.dimension; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import com.yahoo.bard.webservice.druid.serializers.DimensionToDefaultDimensionSpec; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -149,6 +150,15 @@ public interface Dimension { */ String getLongName(); + /** + * Returns the storage strategy of the dimension. + *

+ * See {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy}. + * + * @return the storage strategy of the dimension. + */ + StorageStrategy getStorageStrategy(); + /** * Get the cardinality of the dimension. * diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java index 105acedda7..5ca58b39d6 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/impl/KeyValueStoreDimension.java @@ -9,6 +9,7 @@ import com.yahoo.bard.webservice.data.dimension.DimensionRow; import com.yahoo.bard.webservice.data.dimension.KeyValueStore; import com.yahoo.bard.webservice.data.dimension.SearchProvider; +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy; import com.yahoo.bard.webservice.util.DimensionStoreKeyUtils; import com.fasterxml.jackson.core.type.TypeReference; @@ -65,6 +66,7 @@ public class KeyValueStoreDimension implements Dimension { private final DimensionField key; private final boolean isAggregatable; + private final StorageStrategy storageStrategy; /** * Constructor. @@ -78,6 +80,8 @@ public class KeyValueStoreDimension implements Dimension { * @param searchProvider Search provider over the metadata for the dimension * @param defaultDimensionFields Default fields for the dimension * @param isAggregatable Whether the dimension is aggregatable + * @param storageStrategy Strategy of how dimension is loaded. See + * {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy} */ public KeyValueStoreDimension( String dimensionName, @@ -88,7 +92,8 @@ public KeyValueStoreDimension( @NotNull KeyValueStore keyValueStore, SearchProvider searchProvider, @NotNull LinkedHashSet defaultDimensionFields, - boolean isAggregatable + boolean isAggregatable, + StorageStrategy storageStrategy ) { this.apiName = dimensionName; this.longName = longName; @@ -112,6 +117,7 @@ public KeyValueStoreDimension( this.searchProvider.setKeyValueStore(keyValueStore); this.isAggregatable = isAggregatable; + this.storageStrategy = storageStrategy; } /** @@ -142,7 +148,8 @@ public KeyValueStoreDimension( keyValueStore, searchProvider, new LinkedHashSet<>(), - true + true, + StorageStrategy.LOADED ); } @@ -175,7 +182,8 @@ public KeyValueStoreDimension( keyValueStore, searchProvider, new LinkedHashSet<>(), - isAggregatable + isAggregatable, + StorageStrategy.LOADED ); } @@ -243,7 +251,8 @@ public KeyValueStoreDimension( keyValueStore, searchProvider, new LinkedHashSet<>(), - true + true, + StorageStrategy.LOADED ); this.addAllDimensionRows(dimensionRows); } @@ -263,7 +272,8 @@ public KeyValueStoreDimension(DimensionConfig dimensionConfig) { dimensionConfig.getKeyValueStore(), dimensionConfig.getSearchProvider(), dimensionConfig.getDefaultDimensionFields(), - dimensionConfig.isAggregatable() + dimensionConfig.isAggregatable(), + StorageStrategy.LOADED ); } @@ -344,6 +354,11 @@ public String getLongName() { return longName; } + @Override + public StorageStrategy getStorageStrategy() { + return storageStrategy; + } + @Override public int getCardinality() { return searchProvider.getDimensionCardinality(); @@ -541,6 +556,29 @@ public boolean isAggregatable() { return isAggregatable; } + /** + * Constructs a new KeyValueStoreDimension with specified + * {@link com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy}. + * + * @param storageStrategy The specified StorageStrategy + * + * @return the new KeyValueStoreDimension with the specified StorageStrategy + */ + public KeyValueStoreDimension withStorageStrategy(StorageStrategy storageStrategy) { + return new KeyValueStoreDimension( + apiName, + longName, + category, + description, + dimensionFields, + keyValueStore, + searchProvider, + defaultDimensionFields, + isAggregatable, + storageStrategy + ); + } + @Override public boolean equals(Object o) { if (this == o) { return true; } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/metadata/StorageStrategy.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/metadata/StorageStrategy.java new file mode 100644 index 0000000000..6cc294f6fe --- /dev/null +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/dimension/metadata/StorageStrategy.java @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Inc. +// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. +package com.yahoo.bard.webservice.data.dimension.metadata; + +/** + * Allows clients to be notified if a dimension's values are browsable and searchable. + *

+ * For the non-loaded dimensions(A "non-loaded dimension" is a fact based dimension, where we don't load any domain data + * for it, but simply send queries directly to druid), we need to surface metadata to the UI. If there aren't dimension + * values loaded, we can't validate when we build filters and you can't use the dimension values endpoint to browse + * values. UI needs to know that a dimension isn't going to be validated and searched. The way that UI knows about this + * is through this StorageStrategy + */ +public enum StorageStrategy { + /** + * Loaded dimension. + */ + LOADED, + /** + * Non-loaded dimension. + */ + NONE; + + @Override + public String toString() { + return name(); + } +} diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DimensionsServlet.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DimensionsServlet.java index c0b4dcc51a..79ada7516f 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DimensionsServlet.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DimensionsServlet.java @@ -375,6 +375,7 @@ public static Map getDimensionSummaryView(Dimension dimension, f resultRow.put("longName", dimension.getLongName()); resultRow.put("uri", getDimensionUrl(dimension, uriInfo)); resultRow.put("cardinality", dimension.getCardinality()); + resultRow.put("storageStrategy", dimension.getStorageStrategy()); return resultRow; } @@ -400,6 +401,7 @@ public static Map getDimensionFullView( resultRow.put("fields", dimension.getDimensionFields()); resultRow.put("values", getDimensionValuesUrl(dimension, uriInfo)); resultRow.put("cardinality", dimension.getCardinality()); + resultRow.put("storageStrategy", dimension.getStorageStrategy()); resultRow.put( "tables", TablesServlet.getLogicalTableListSummaryView( diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToNameSerializerSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToNameSerializerSpec.groovy index c2a0b50579..7068bf7c9f 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToNameSerializerSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToNameSerializerSpec.groovy @@ -7,6 +7,7 @@ import com.yahoo.bard.webservice.data.dimension.Dimension import com.yahoo.bard.webservice.data.dimension.DimensionField import com.yahoo.bard.webservice.data.dimension.DimensionRow import com.yahoo.bard.webservice.data.dimension.SearchProvider +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.databind.ObjectMapper @@ -30,100 +31,105 @@ class DimensionToNameSerializerSpec extends Specification { } @JsonSerialize - public static class DummyDimension implements Dimension { + static class DummyDimension implements Dimension { @JsonValue - public String example() { - return "woot"; + String example() { + return "woot" } @Override - public void setLastUpdated(DateTime lastUpdated) { + void setLastUpdated(DateTime lastUpdated) { } @Override - public String getApiName() { - return "abc"; + String getApiName() { + return "abc" } @Override - public String getDescription() { - return null; + String getDescription() { + return null } @Override - public DateTime getLastUpdated() { - return null; + DateTime getLastUpdated() { + return null } @Override - public LinkedHashSet getDimensionFields() { - return null; + LinkedHashSet getDimensionFields() { + return null } @Override - public LinkedHashSet getDefaultDimensionFields() { - return null; + LinkedHashSet getDefaultDimensionFields() { + return null } @Override - public DimensionField getFieldByName(String name) { - return null; + DimensionField getFieldByName(String name) { + return null } @Override - public SearchProvider getSearchProvider() { - return null; + SearchProvider getSearchProvider() { + return null } @Override - public void addDimensionRow(DimensionRow dimensionRow) { + void addDimensionRow(DimensionRow dimensionRow) { } @Override - public void addAllDimensionRows(Set dimensionRows) { + void addAllDimensionRows(Set dimensionRows) { } @Override - public DimensionRow findDimensionRowByKeyValue(String value) { - return null; + DimensionRow findDimensionRowByKeyValue(String value) { + return null } @Override - public DimensionField getKey() { - return null; + DimensionField getKey() { + return null } @Override - public DimensionRow parseDimensionRow(Map fieldNameValueMap) { - return null; + DimensionRow parseDimensionRow(Map fieldNameValueMap) { + return null } @Override - public DimensionRow createEmptyDimensionRow(String keyFieldValue) { - return null; + DimensionRow createEmptyDimensionRow(String keyFieldValue) { + return null } @Override - public String getCategory() { - return null; + String getCategory() { + return null } @Override - public String getLongName() { - return null; + String getLongName() { + return null } @Override - public int getCardinality() { - return 0; + int getCardinality() { + return 0 } @Override - public boolean isAggregatable() { - return false; + boolean isAggregatable() { + return false + } + + @Override + StorageStrategy getStorageStrategy() { + return null } } } diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/AggregatabilityValidationSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/AggregatabilityValidationSpec.groovy index cc86447347..64148d60cb 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/AggregatabilityValidationSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/AggregatabilityValidationSpec.groovy @@ -11,6 +11,7 @@ import com.yahoo.bard.webservice.data.dimension.DimensionField import com.yahoo.bard.webservice.data.dimension.MapStoreManager import com.yahoo.bard.webservice.data.dimension.impl.KeyValueStoreDimension import com.yahoo.bard.webservice.data.dimension.impl.ScanSearchProviderManager +import com.yahoo.bard.webservice.data.dimension.metadata.StorageStrategy import com.yahoo.bard.webservice.data.metric.MetricDictionary import com.yahoo.bard.webservice.table.LogicalTable import com.yahoo.bard.webservice.table.TableGroup @@ -62,7 +63,8 @@ class AggregatabilityValidationSpec extends Specification { MapStoreManager.getInstance(name), ScanSearchProviderManager.getInstance(name), new LinkedHashSet(), - false + false, + StorageStrategy.LOADED ) keyValueStoreDimension.setLastUpdated(new DateTime(10000)) dimensionDict.add(keyValueStoreDimension) diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy index ff02e1b320..74b9313043 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/DimensionsServletComponentSpec.groovy @@ -80,14 +80,14 @@ class DimensionsServletComponentSpec extends Specification { String expectedResponse = """{ "dimensions": [ - {"category": "General", "name": "color", "longName": "color", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/color", "cardinality": 0}, - {"category": "General", "name": "shape", "longName": "shape", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/shape", "cardinality": 38}, - {"category": "General", "name": "size", "longName": "size", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/size", "cardinality": 0}, - {"category": "General", "name": "model", "longName": "model", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/model", "cardinality": 21}, - {"category": "General", "name": "other", "longName": "other", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/other", "cardinality": 100000}, - {"category": "General", "name": "sex", "longName": "sex", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/sex", "cardinality": 0}, - {"category": "General", "name": "species", "longName": "species", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/species", "cardinality": 0}, - {"category": "General", "name": "breed", "longName": "breed", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/breed", "cardinality": 0} + {"category": "General", "name": "color", "longName": "color", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/color", "cardinality": 0, "storageStrategy":"LOADED"}, + {"category": "General", "name": "shape", "longName": "shape", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/shape", "cardinality": 38, "storageStrategy":"LOADED"}, + {"category": "General", "name": "size", "longName": "size", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/size", "cardinality": 0, "storageStrategy":"LOADED"}, + {"category": "General", "name": "model", "longName": "model", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/model", "cardinality": 21, "storageStrategy":"LOADED"}, + {"category": "General", "name": "other", "longName": "other", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/other", "cardinality": 100000, "storageStrategy":"LOADED"}, + {"category": "General", "name": "sex", "longName": "sex", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/sex", "cardinality": 0, "storageStrategy":"LOADED"}, + {"category": "General", "name": "species", "longName": "species", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/species", "cardinality": 0, "storageStrategy":"LOADED"}, + {"category": "General", "name": "breed", "longName": "breed", "uri": "http://localhost:${jtb.getHarness().getPort()}/dimensions/breed", "cardinality": 0, "storageStrategy":"LOADED"} ] }""" @@ -115,6 +115,7 @@ class DimensionsServletComponentSpec extends Specification { ], "longName": "other", "name": "other", + "storageStrategy":"LOADED", "tables": [ { "category": "General", diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy index ceea11e13d..f77b904abd 100644 --- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy +++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy @@ -114,6 +114,7 @@ class TablesServletSpec extends Specification { "name": "$it", "longName": "$it", "cardinality": 0, + "storageStrategy":"LOADED", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/$it" }""" } diff --git a/fili-wikipedia-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy b/fili-wikipedia-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy index 3291b3df18..e522cd497e 100644 --- a/fili-wikipedia-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy +++ b/fili-wikipedia-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/TablesServletSpec.groovy @@ -121,6 +121,7 @@ class TablesServletSpec extends Specification { "name": "$it", "longName": "wiki $it", "cardinality": 0, + "storageStrategy":"LOADED", "uri": "http://localhost:${jerseyTestBinder.getHarness().getPort()}/dimensions/$it" }""" }