Implement DruidPartialDataResponseProcessor (#275)

yahoo · May 30, 2017 · f9db95a · f9db95a
1 parent fa5aa1f
commit f9db95a
Show file tree

Hide file tree

Showing 10 changed files with 616 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,11 @@ Current
 -------
 ### Added:
 
+- [Implement DruidPartialDataResponseProcessor](https://github.com/yahoo/fili/pull/275)
+    * Add `FullResponseProcessor` interface that extends `ResponseProcessor`
+    * Add response status code to JSON response
+    * Add `DruidPartialDataResponseProcessor` that checks for any missing data that's not being found
+
 - [Add `DataSourceName` concept, removing responsibility from `TableName`](https://github.com/yahoo/fili/pull/263)
     * `TableName` was serving double-duty, and it was causing problems and confusion. Splitting the concepts fixes it.
 

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/client/HttpErrorCallback.java b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/client/HttpErrorCallback.java
@@ -14,18 +14,18 @@ public interface HttpErrorCallback {
     /**
      * Invoke the error callback code.
      *
+     * @param statusCode  Http status code of the error response
      * @param reasonPhrase  The reason for the error. Often the status code description.
      * @param responseBody  The body of the error response
-     * @param statusCode  Http status code of the error response
      */
     void invoke(int statusCode, String reasonPhrase, String responseBody);
 
     /**
      * Stop the request timer, start the response timer, and then invoke the error callback code.
      *
+     * @param statusCode  Http status code of the error response
      * @param reasonPhrase  The reason for the error. Often the status code description.
      * @param responseBody  The body of the error response
-     * @param statusCode  Http status code of the error response
      */
     default void dispatch(int statusCode, String reasonPhrase, String responseBody) {
         RequestLog.stopTiming(REQUEST_WORKFLOW_TIMER);

diff --git a/...in/java/com/yahoo/bard/webservice/druid/client/impl/HeaderNestingJsonBuilderStrategy.java b/...in/java/com/yahoo/bard/webservice/druid/client/impl/HeaderNestingJsonBuilderStrategy.java
@@ -2,6 +2,8 @@
 // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
 package com.yahoo.bard.webservice.druid.client.impl;
 
+import com.yahoo.bard.webservice.web.responseprocessors.DruidJsonResponseContentKeys;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
@@ -30,15 +32,22 @@ public HeaderNestingJsonBuilderStrategy(Function<Response, JsonNode> baseStrateg
 
     @Override
     public JsonNode apply(Response response) {
+        MappingJsonFactory mappingJsonFactory = new MappingJsonFactory();
         ObjectNode objectNode = JsonNodeFactory.instance.objectNode();
-        objectNode.set("response", baseStrategy.apply(response));
+        objectNode.set(DruidJsonResponseContentKeys.RESPONSE.getName(), baseStrategy.apply(response));
         try {
             objectNode.set(
-                    "X-Druid-Response-Context",
-                    new MappingJsonFactory()
-                            .createParser(response.getHeader("X-Druid-Response-Context"))
+                    DruidJsonResponseContentKeys.DRUID_RESPONSE_CONTEXT.getName(),
+                    mappingJsonFactory
+                            .createParser(
+                                    response.getHeader(DruidJsonResponseContentKeys.DRUID_RESPONSE_CONTEXT.getName())
+                            )
                             .readValueAsTree()
             );
+            objectNode.set(
+                    DruidJsonResponseContentKeys.STATUS_CODE.getName(),
+                    mappingJsonFactory.createParser(String.valueOf(response.getStatusCode())).readValueAsTree()
+            );
         } catch (IOException ioe) {
             throw new IllegalStateException(ioe);
         }

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/ErrorMessageFormat.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/ErrorMessageFormat.java
@@ -216,22 +216,35 @@ public enum ErrorMessageFormat implements MessageFormatter {
     RESULT_MAPPING_FAILURE(
             "Error occurred while processing response data: %s"
     ),
+
     INVALID_DATASOURCE_UNION(
             "Union Data Source had conflicting name mappings for logical dimension '%s' with mappings of '%s' and '%s'"
     ),
 
-
     DATA_AVAILABILITY_MISMATCH(
             "Data availability expectation does not match with actual query result obtained from druid for the " +
                     "following intervals %s where druid does not have data"
     ),
 
-    TOO_MUCH_INTERVAL_MISSING(
-            "More than %s interval missing information received from druid, inspect if query " +
-                    "expects more than %s missing intervals or increase " +
-                    "uncoveredIntervalsLimit configuration value"
+    TOO_MANY_INTERVALS_MISSING(
+            "Query is returning more than the configured limit of '%s' missing intervals. " +
+                    "There may be a problem with your data."
     ),
 
+    CONTEXT_AND_STATUS_MISSING_FROM_RESPONSE("JSON response is missing X-Druid-Response-Context and status code"),
+
+    DRUID_RESPONSE_CONTEXT_MISSING_FROM_RESPONSE("JSON response is missing X-Druid-Response-Context"),
+
+    UNCOVERED_INTERVALS_MISSING_FROM_RESPONSE(
+            "JSON response is missing 'uncoveredIntervals' from X-Druid-Response-Context header"
+    ),
+
+    UNCOVERED_INTERVALS_OVERFLOWED_MISSING_FROM_RESPONSE(
+            "JSON response is missing 'uncoveredIntervalsOverflowed' from X-Druid-Response-Context header"
+    ),
+
+    STATUS_CODE_MISSING_FROM_RESPONSE("JSON response is missing response status code"),
+
     TOO_MANY_BACKING_DATA_SOURCES("TableDataSource built with too many backing data sources: %s"),
     TOO_FEW_BACKING_DATA_SOURCES("TableDataSource built with insufficient backing data sources: %s")
     ;

diff --git a/...n/java/com/yahoo/bard/webservice/web/responseprocessors/DruidJsonResponseContentKeys.java b/...n/java/com/yahoo/bard/webservice/web/responseprocessors/DruidJsonResponseContentKeys.java
@@ -0,0 +1,30 @@
+// Copyright 2017 Yahoo Inc.
+// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
+package com.yahoo.bard.webservice.web.responseprocessors;
+
+/**
+ * Enumerates the list of keys expected to be found in the FullResponseProcessor.
+ */
+public enum DruidJsonResponseContentKeys {
+    DRUID_RESPONSE_CONTEXT("X-Druid-Response-Context"),
+    UNCOVERED_INTERVALS("uncoveredIntervals"),
+    UNCOVERED_INTERVALS_OVERFLOWED("uncoveredIntervalsOverflowed"),
+    STATUS_CODE("status-code"),
+    RESPONSE("response")
+    ;
+
+    private final String name;
+
+    /**
+     * Constructor.
+     *
+     * @param name  Name of the context key
+     */
+    DruidJsonResponseContentKeys(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}