Add uncovered intervals concept (#264)

Prep for Partial Data v2 extension
yahoo · May 9, 2017 · 8e3d7f4 · 8e3d7f4
1 parent 509dc1e
commit 8e3d7f4
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,11 @@ Current
 -------
 ### Added:
 
+- [Prepare For Partial Data V2](https://github.com/yahoo/fili/pull/264)
+    * Add new query context for druid's uncovered interval feature
+    * Add a configurable property named "druid_uncovered_interval_limit"
+    * Add new response error messages as needed by Partial Data V2
+
 - [Add MetricUnionCompositeTableDefinition](https://github.com/yahoo/fili/pull/258)
 
 - [Add partition availability and table](https://github.com/yahoo/fili/pull/244)

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/query/QueryContext.java b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/query/QueryContext.java
@@ -8,6 +8,7 @@
 import static com.yahoo.bard.webservice.druid.model.query.QueryContext.Param.PRIORITY;
 import static com.yahoo.bard.webservice.druid.model.query.QueryContext.Param.QUERY_ID;
 import static com.yahoo.bard.webservice.druid.model.query.QueryContext.Param.TIMEOUT;
+import static com.yahoo.bard.webservice.druid.model.query.QueryContext.Param.UNCOVERED_INTERVALS_LIMIT;
 import static com.yahoo.bard.webservice.druid.model.query.QueryContext.Param.USE_CACHE;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -36,7 +37,9 @@ public enum Param {
         USE_CACHE("useCache"),
         POPULATE_CACHE("populateCache"),
         BY_SEGMENT("bySegment"),
-        FINALIZE("finalize");
+        FINALIZE("finalize"),
+        UNCOVERED_INTERVALS_LIMIT("uncoveredIntervalsLimit")
+        ;
 
         private final String jsonName;
 
@@ -63,7 +66,8 @@ String getName() {
             new SimpleImmutableEntry<>(USE_CACHE, Boolean.class),
             new SimpleImmutableEntry<>(POPULATE_CACHE, Boolean.class),
             new SimpleImmutableEntry<>(BY_SEGMENT, Boolean.class),
-            new SimpleImmutableEntry<>(FINALIZE, Boolean.class))
+            new SimpleImmutableEntry<>(FINALIZE, Boolean.class),
+            new SimpleImmutableEntry<>(UNCOVERED_INTERVALS_LIMIT, Number.class))
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
 
     @JsonIgnore
@@ -208,6 +212,10 @@ public QueryContext withBySegment(Boolean bySegment) {
     public QueryContext withFinalize(Boolean finalize) {
         return withValue(FINALIZE, finalize);
     }
+
+    public QueryContext withUncoveredIntervalsLimit(Integer uncoveredIntervalsLimit) {
+        return withValue(UNCOVERED_INTERVALS_LIMIT, uncoveredIntervalsLimit);
+    }
     // CHECKSTYLE:ON
 
     @JsonIgnore
@@ -255,6 +263,11 @@ public Boolean getFinalize() {
         return (Boolean) contextMap.get(FINALIZE);
     }
 
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public Integer getUncoveredIntervalsLimit() {
+        return (Integer) contextMap.get(UNCOVERED_INTERVALS_LIMIT);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o instanceof QueryContext) {

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/ErrorMessageFormat.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/ErrorMessageFormat.java
@@ -215,6 +215,17 @@ public enum ErrorMessageFormat implements MessageFormatter {
 
     RESULT_MAPPING_FAILURE(
             "Error occurred while processing response data: %s"
+    ),
+
+    DATA_AVAILABILITY_MISMATCH(
+            "Data availability expectation does not match with actual query result obtained from druid for the " +
+                    "following intervals %s where druid does not have data"
+    ),
+
+    TOO_MUCH_INTERVAL_MISSING(
+            "More than %s interval missing information received from druid, inspect if query " +
+                    "expects more than %s missing intervals or increase " +
+                    "uncoveredIntervalsLimit configuration value"
     )
     ;
 

diff --git a/fili-core/src/main/resources/moduleConfig.properties b/fili-core/src/main/resources/moduleConfig.properties
@@ -124,6 +124,10 @@ bard__default_asyncAfter=never
 # Flag to turn on case sensitive keys in keyvalue store
 bard__case_sensitive_keys_enabled = false
 
+# Sets the upper limit of the number of Druid uncovered interval. Default is 0, which means
+# no uncovered interval is allowed
+bard__druid_uncovered_interval_limit = 0
+
 # The implementation of the com.yahoo.bard.webservice.logging.LogFormatter to use to format the RequestLog logging
 # blocks. By default, the RequestLog is formatted as JSON.
 bard__log_formatter_implementation=com.yahoo.bard.webservice.logging.JsonLogFormatter
diff --git a/fili-core/src/test/resources/testApplicationConfig.properties b/fili-core/src/test/resources/testApplicationConfig.properties
@@ -42,7 +42,6 @@ bard__redis_namespace = test
 # The channel on which RedisBroadcastChannel can publish/listen to messages
 bard__redisbroadcastchannel_name = broadcast
 
-
 # Don't start Partial data loading during tests
 bard__druid_seg_loader_timer_delay = 60000
 
@@ -60,7 +59,6 @@ bard__lucene_index_path = ./target/tmp/
 # data endpoint does not paginate by default.
 bard__default_per_page = 10000
 
-
 # Intersection reporting enabled or not.
 bard__intersection_reporting_enabled = false