From 62e7525079b1261baf5b0c6985faa27bd8c329d6 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Fri, 26 May 2017 12:54:00 -0500
Subject: [PATCH 01/12] DruidNavigator locked up while loading several tables

It appears the locking mechanism was forced to time out while loading
multiple tables
---
 .../data/config/auto/DruidNavigator.java      | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
index de679509ed..065aae3b84 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
@@ -69,7 +69,7 @@ public List<? extends DataSourceConfiguration> get() {
      * ["wikiticker"]
      */
     private void loadAllDatasources() {
-        queryDruid(rootNode -> {
+        Future<Response> responseFuture = queryDruid(rootNode -> {
             if (rootNode.isArray()) {
                 rootNode.forEach(jsonNode -> {
                     TableConfig tableConfig = new TableConfig(jsonNode.asText());
@@ -78,6 +78,14 @@ private void loadAllDatasources() {
                 });
             }
         }, COORDINATOR_TABLES_PATH);
+
+        try {
+            //calling get so we wait until responses are loaded before returning and processing continues
+            responseFuture.get(60, TimeUnit.SECONDS);
+        } catch (TimeoutException | InterruptedException | ExecutionException e) {
+            LOG.error("Interrupted while waiting for a response from druid", e);
+            throw new RuntimeException("Unable to automatically configure correctly, no response from druid.", e);
+        }
     }
 
     /**
@@ -193,9 +201,9 @@ private void loadTimeGrains(TableConfig tableConfig, JsonNode segmentJson) {
      * @param successCallback  The callback to be done if the query succeeds.
      * @param url  The url to send the query to.
      */
-    private void queryDruid(SuccessCallback successCallback, String url) {
+    private Future<Response> queryDruid(SuccessCallback successCallback, String url) {
         LOG.debug("Fetching " + url);
-        Future<Response> responseFuture = druidWebService.getJsonObject(
+        return druidWebService.getJsonObject(
                 rootNode -> {
                     LOG.debug("Succesfully fetched " + url);
                     successCallback.invoke(rootNode);
@@ -210,13 +218,5 @@ private void queryDruid(SuccessCallback successCallback, String url) {
                 },
                 url
         );
-
-        try {
-            //calling get so we wait until responses are loaded before returning and processing continues
-            responseFuture.get(30, TimeUnit.SECONDS);
-        } catch (TimeoutException | InterruptedException | ExecutionException e) {
-            LOG.error("Interrupted while waiting for a response from druid", e);
-            throw new RuntimeException("Unable to automatically configure correctly, no response from druid.", e);
-        }
     }
 }

From 8f62462e4cef70f22271902f7670c69b6eb3428f Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Fri, 26 May 2017 13:22:33 -0500
Subject: [PATCH 02/12] Fix javadoc, revert timeout to 30 seconds

---
 .../wiki/webservice/data/config/auto/DruidNavigator.java      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
index 065aae3b84..93ea89ad8f 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
@@ -81,7 +81,7 @@ private void loadAllDatasources() {
 
         try {
             //calling get so we wait until responses are loaded before returning and processing continues
-            responseFuture.get(60, TimeUnit.SECONDS);
+            responseFuture.get(30, TimeUnit.SECONDS);
         } catch (TimeoutException | InterruptedException | ExecutionException e) {
             LOG.error("Interrupted while waiting for a response from druid", e);
             throw new RuntimeException("Unable to automatically configure correctly, no response from druid.", e);
@@ -200,6 +200,8 @@ private void loadTimeGrains(TableConfig tableConfig, JsonNode segmentJson) {
      *
      * @param successCallback  The callback to be done if the query succeeds.
      * @param url  The url to send the query to.
+     *
+     * @return future response for the druid query
      */
     private Future<Response> queryDruid(SuccessCallback successCallback, String url) {
         LOG.debug("Fetching " + url);

From 3646fc13d6e2f255cf2b96df7d4155336bbe2f83 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Tue, 30 May 2017 09:08:54 -0500
Subject: [PATCH 03/12] Fix DruidNavigator loading multiple tables

- This also adds a test to make sure that multiple tables get configured properly.
- There was also an issue with the GenericTableLoader showing the same metrics,
   granularities, etc for all tables
---
 .../data/config/auto/DruidNavigator.java      | 24 ++++--
 .../data/config/table/GenericTableLoader.java | 54 +++++++------
 .../AutomaticDruidConfigLoaderSpec.groovy     |  3 +-
 ...ipleDatasourceDruidConfigLoaderSpec.groovy | 80 +++++++++++++++++++
 4 files changed, 126 insertions(+), 35 deletions(-)
 create mode 100644 fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
index 93ea89ad8f..f3758101b7 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
@@ -2,14 +2,12 @@
 // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
 package com.yahoo.wiki.webservice.data.config.auto;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.bard.webservice.data.time.DefaultTimeGrain;
 import com.yahoo.bard.webservice.data.time.TimeGrain;
 import com.yahoo.bard.webservice.druid.client.DruidWebService;
 import com.yahoo.bard.webservice.druid.client.SuccessCallback;
 import com.yahoo.bard.webservice.util.IntervalUtils;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
 import org.asynchttpclient.Response;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -69,23 +67,35 @@ public List<? extends DataSourceConfiguration> get() {
      * ["wikiticker"]
      */
     private void loadAllDatasources() {
+        final List<Future<Response>> fullTableResponses = new ArrayList<>();
         Future<Response> responseFuture = queryDruid(rootNode -> {
             if (rootNode.isArray()) {
                 rootNode.forEach(jsonNode -> {
                     TableConfig tableConfig = new TableConfig(jsonNode.asText());
-                    loadTable(tableConfig);
+                    Future<Response> tableResponseFuture = loadTable(tableConfig);
+                    fullTableResponses.add(tableResponseFuture);
                     tableConfigurations.add(tableConfig);
                 });
             }
         }, COORDINATOR_TABLES_PATH);
 
+        // wait until list of table names are loaded before processing continues (i.e. ["t1","t2"])
         try {
-            //calling get so we wait until responses are loaded before returning and processing continues
             responseFuture.get(30, TimeUnit.SECONDS);
         } catch (TimeoutException | InterruptedException | ExecutionException e) {
             LOG.error("Interrupted while waiting for a response from druid", e);
             throw new RuntimeException("Unable to automatically configure correctly, no response from druid.", e);
         }
+
+        // force each individual table to finish loading (i.e. loadTable(t1) and loadTable(t2) have finished)
+        fullTableResponses.forEach(future -> {
+            try {
+                future.get(30,TimeUnit.SECONDS);
+            } catch (InterruptedException | ExecutionException | TimeoutException e) {
+                LOG.error("Interrupted while building tables", e);
+                throw new RuntimeException("Unable to automatically configure correctly, couldn't fetch table data.", e);
+            }
+        });
     }
 
     /**
@@ -108,10 +118,10 @@ private void loadAllDatasources() {
      *
      * @param table The TableConfig to be loaded with queries against druid.
      */
-    private void loadTable(TableConfig table) {
+    private Future<Response> loadTable(TableConfig table) {
         String url = COORDINATOR_TABLES_PATH + table.getName() + "/?full";
         String segmentsPath = "segments";
-        queryDruid(rootNode -> {
+        return queryDruid(rootNode -> {
             if (rootNode.get(segmentsPath).size() == 0) {
                 LOG.error("The segments list returned from {} was empty.", url);
                 throw new RuntimeException("Can't configure table without segment data.");
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
index 0b59e56cfd..29ceed1ecb 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
@@ -21,29 +21,23 @@
 import com.yahoo.wiki.webservice.data.config.dimension.GenericDimensionConfigs;
 import com.yahoo.wiki.webservice.data.config.metric.DruidMetricName;
 import com.yahoo.wiki.webservice.data.config.metric.FiliApiMetricName;
-
 import org.joda.time.DateTimeZone;
 
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
+import javax.validation.constraints.NotNull;
+import java.util.*;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import javax.validation.constraints.NotNull;
-
 /**
  * Load the table configuration for any druid setup.
  */
 public class GenericTableLoader extends BaseTableLoader {
-    private Set<Granularity> validGrains = new HashSet<>();
+    private Map<String,Set<Granularity>> validGrains = new HashMap<>();
     // Set up the metrics
-    private Set<FieldName> druidMetricNames = new HashSet<>();
-    private Set<ApiMetricName> apiMetricNames = new HashSet<>();
+    private Map<String,Set<FieldName>> druidMetricNames = new HashMap<>();
+    private Map<String,Set<ApiMetricName>> apiMetricNames = new HashMap<>();
     // Set up the table definitions
-    private Set<PhysicalTableDefinition> tableDefinitions = new HashSet<>();
+    private Map<String, Set<PhysicalTableDefinition>> tableDefinitions = new HashMap<>();
     private final Supplier<List<? extends DataSourceConfiguration>> configLoader;
 
     /**
@@ -71,23 +65,28 @@ public GenericTableLoader(
     private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
         configLoader.get().forEach(dataSourceConfiguration -> {
 
-            druidMetricNames = dataSourceConfiguration.getMetrics()
+            druidMetricNames.put( dataSourceConfiguration.getName(),
+                    dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(DruidMetricName::new)
-                    .collect(Collectors.toSet());
+                    .collect(Collectors.toSet()));
 
-            tableDefinitions = getPhysicalTableDefinitions(
-                    dataSourceConfiguration,
-                    dataSourceConfiguration.getValidTimeGrain(),
-                    genericDimensionConfigs.getAllDimensionConfigurations()
+            tableDefinitions.put(dataSourceConfiguration.getName(),
+                    getPhysicalTableDefinitions(
+                            dataSourceConfiguration,
+                            dataSourceConfiguration.getValidTimeGrain(),
+                            genericDimensionConfigs.getAllDimensionConfigurations()
+                    )
             );
 
-            apiMetricNames = dataSourceConfiguration.getMetrics()
+            apiMetricNames.put(dataSourceConfiguration.getName(),
+                    dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(metricName -> new FiliApiMetricName(metricName, dataSourceConfiguration.getValidTimeGrain()))
-                    .collect(Collectors.toSet());
+                    .collect(Collectors.toSet())
+            );
 
-            validGrains = getGranularities(dataSourceConfiguration);
+            validGrains.put(dataSourceConfiguration.getName(), getGranularities(dataSourceConfiguration));
         });
 
     }
@@ -129,7 +128,7 @@ private Set<PhysicalTableDefinition> getPhysicalTableDefinitions(
                         new ConcretePhysicalTableDefinition(
                                 dataSourceConfiguration.getTableName(),
                                 zonedTimeGrain,
-                                druidMetricNames,
+                                druidMetricNames.get(dataSourceConfiguration.getName()),
                                 dimsBasefactDruidTable
                         )
                 )
@@ -145,19 +144,22 @@ private Set<PhysicalTableDefinition> getPhysicalTableDefinitions(
     public void loadTableDictionary(ResourceDictionaries dictionaries) {
         configLoader.get()
                 .forEach(table -> {
-                    Set<TableName> currentTableGroupTableNames = tableDefinitions.stream()
+                    Set<TableName> currentTableGroupTableNames = tableDefinitions.get(table.getName())
+                            .stream()
                             .map(PhysicalTableDefinition::getName)
                             .collect(Collectors.toSet());
+
                     TableGroup tableGroup = buildDimensionSpanningTableGroup(
                             currentTableGroupTableNames,
-                            tableDefinitions,
+                            tableDefinitions.get(table.getName()),
                             dictionaries,
-                            apiMetricNames
+                            apiMetricNames.get(table.getName())
                     );
+
                     loadLogicalTableWithGranularities(
                             table.getTableName().asName(),
                             tableGroup,
-                            validGrains,
+                            validGrains.get(table.getName()),
                             dictionaries
                     );
                 });
diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
index d1d057fcfa..0ed2d72afd 100644
--- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
@@ -5,7 +5,6 @@ import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService
 import com.yahoo.wiki.webservice.data.config.auto.DataSourceConfiguration
 import com.yahoo.wiki.webservice.data.config.auto.DruidNavigator
 import com.yahoo.wiki.webservice.data.config.auto.TableConfig
-
 import spock.lang.Specification
 
 public class AutomaticDruidConfigLoaderSpec extends Specification {
@@ -53,7 +52,7 @@ public class AutomaticDruidConfigLoaderSpec extends Specification {
             } else if (druidWebService.lastUrl == '/datasources/' + datasource + "/?full") {
                 return expectedMetricsAndDimensions
             }
-            return "Unexpected URL"
+            println "No response for " + druidWebService.lastUrl
         }
     }
 
diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy
new file mode 100644
index 0000000000..86df7170b5
--- /dev/null
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy
@@ -0,0 +1,80 @@
+package com.yahoo.wiki.webservice.web.endpoints
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService
+import com.yahoo.wiki.webservice.data.config.auto.DataSourceConfiguration
+import com.yahoo.wiki.webservice.data.config.auto.DruidNavigator
+import spock.lang.Specification
+
+class MultipleDatasourceDruidConfigLoaderSpec extends Specification {
+    private String[] datasources = ["table1", "table2"]
+    private String[] table1_metrics = ["1_metric1", "1_metric2"]
+    private String[] table1_dimensions = ["1_dim1", "1_dim2"]
+    private String[] table2_metrics = ["2_metric1", "2_metric2"]
+    private String[] table2_dimensions = ["2_dim1", "2_dim2"]
+
+    private TestDruidWebService druidWebService
+
+    def setup() {
+        druidWebService = new TestDruidWebService("testInstance")
+        druidWebService.jsonResponse = {
+            if (druidWebService.lastUrl.equals("/datasources/")) {
+                return new ObjectMapper().writeValueAsString(datasources)
+            } else if (druidWebService.lastUrl.equals("/datasources/table1/?full")) {
+                return getFullTable(datasources[0], table1_metrics, table1_dimensions)
+            } else if (druidWebService.lastUrl.equals("/datasources/table2/?full")) {
+                return getFullTable(datasources[1], table2_metrics, table2_dimensions)
+            }
+            println "No response for " + druidWebService.lastUrl
+        }
+    }
+
+    def "Load Multiple datasources"() {
+        when: "We query druid"
+        DruidNavigator druidNavigator = new DruidNavigator(druidWebService)
+
+        then: "What we expect"
+        List<DataSourceConfiguration> tables = druidNavigator.get();
+        tables.size() == 2
+        DataSourceConfiguration table1 = tables.get(0)
+        table1.tableName.asName() == datasources[0]
+        table1.metrics.containsAll(table1_metrics.toList())
+        table1.dimensions.containsAll(table1_dimensions.toList())
+
+        DataSourceConfiguration table2 = tables.get(1)
+        table2.tableName.asName() == datasources[1]
+        table2.metrics.containsAll(table2_metrics.toList())
+        table2.dimensions.containsAll(table2_dimensions.toList())
+    }
+
+
+    private static String getFullTable(String name, String[] metrics, String[] dimensions) {
+        return """{
+    "name": "${name}",
+    "properties": {},
+    "segments": [
+        {
+            "dataSource": "${name}",
+            "interval": "2015-09-12T00:00:00.000Z/2015-09-13T00:00:00.000Z",
+            "version": "2017-02-27T03:06:09.422Z",
+            "loadSpec": 
+                {
+                    "type": "local",
+                    "path": "2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z/2017-02-27T03:06:09.422Z/0/index.zip"
+                },
+            "dimensions": "${dimensions.join(",")}",
+            "metrics":  "${metrics.join(",")}",
+            "shardSpec": 
+                {
+                    "type": "none"
+                },
+            "binaryVersion": 9,
+            "size": 5537610,
+            "identifier": "${name}_2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z_2017-02-27T03:06:09.422Z"
+        }
+    ]
+}
+
+"""
+    }
+}

From a996d2fdd1f2dadb0f2d1d5054b6e32620f6d8c3 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Tue, 30 May 2017 11:35:20 -0500
Subject: [PATCH 04/12] Fix issue where multiple tables load the same dimension
 name

---
 .../data/config/dimension/GenericDimensionConfigs.java           | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
index 1f6d9794b7..6812360ec1 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
@@ -34,6 +34,7 @@ public class GenericDimensionConfigs {
     public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>> configLoader) {
         dimensionConfigs = configLoader.get().stream()
                 .flatMap(tableName -> tableName.getDimensions().stream())
+                .distinct()
                 .map(dimensionName -> new DefaultKeyValueStoreDimensionConfig(
                         () -> dimensionName,
                         dimensionName,

From bb282ab30175d7f003b15eb9c27d9c4dfa62f53d Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Tue, 30 May 2017 11:47:39 -0500
Subject: [PATCH 05/12] Fix check style errors

---
 .../wiki/webservice/data/config/auto/DruidNavigator.java  | 6 ++++--
 .../webservice/data/config/table/GenericTableLoader.java  | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
index f3758101b7..147d4d705a 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
@@ -90,10 +90,10 @@ private void loadAllDatasources() {
         // force each individual table to finish loading (i.e. loadTable(t1) and loadTable(t2) have finished)
         fullTableResponses.forEach(future -> {
             try {
-                future.get(30,TimeUnit.SECONDS);
+                future.get(30, TimeUnit.SECONDS);
             } catch (InterruptedException | ExecutionException | TimeoutException e) {
                 LOG.error("Interrupted while building tables", e);
-                throw new RuntimeException("Unable to automatically configure correctly, couldn't fetch table data.", e);
+                throw new RuntimeException("Unable to configure, couldn't fetch table data.", e);
             }
         });
     }
@@ -117,6 +117,8 @@ private void loadAllDatasources() {
      * }
      *
      * @param table The TableConfig to be loaded with queries against druid.
+     *
+     * @return future response for the query loading the table
      */
     private Future<Response> loadTable(TableConfig table) {
         String url = COORDINATOR_TABLES_PATH + table.getName() + "/?full";
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
index 29ceed1ecb..0e0b7b0269 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
@@ -32,10 +32,10 @@
  * Load the table configuration for any druid setup.
  */
 public class GenericTableLoader extends BaseTableLoader {
-    private Map<String,Set<Granularity>> validGrains = new HashMap<>();
+    private Map<String, Set<Granularity>> validGrains = new HashMap<>();
     // Set up the metrics
-    private Map<String,Set<FieldName>> druidMetricNames = new HashMap<>();
-    private Map<String,Set<ApiMetricName>> apiMetricNames = new HashMap<>();
+    private Map<String, Set<FieldName>> druidMetricNames = new HashMap<>();
+    private Map<String, Set<ApiMetricName>> apiMetricNames = new HashMap<>();
     // Set up the table definitions
     private Map<String, Set<PhysicalTableDefinition>> tableDefinitions = new HashMap<>();
     private final Supplier<List<? extends DataSourceConfiguration>> configLoader;
@@ -65,7 +65,7 @@ public GenericTableLoader(
     private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
         configLoader.get().forEach(dataSourceConfiguration -> {
 
-            druidMetricNames.put( dataSourceConfiguration.getName(),
+            druidMetricNames.put(dataSourceConfiguration.getName(),
                     dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(DruidMetricName::new)

From 455c25e004596738bf15eaf4c4e6130a118d2ead Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Tue, 30 May 2017 13:23:24 -0500
Subject: [PATCH 06/12] Fix bug where all dimensions displayed for all tables.

Now dimensions show up only in the correct table.
Also made a few fields final.
---
 .../dimension/GenericDimensionConfigs.java    | 62 ++++++++++++-------
 .../config/metric/GenericMetricLoader.java    |  2 +-
 .../data/config/table/GenericTableLoader.java | 13 ++--
 .../ConfigurationLoaderSpec.groovy            |  2 +-
 4 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
index 6812360ec1..612f310895 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
@@ -17,6 +17,8 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -24,7 +26,7 @@
  * Hold all the dimension configurations for a generic druid configuration.
  */
 public class GenericDimensionConfigs {
-    private final Set<DimensionConfig> dimensionConfigs;
+    private final Map<String, Set<DimensionConfig>> dimensionConfigs;
 
     /**
      * Construct the dimension configurations.
@@ -32,25 +34,29 @@ public class GenericDimensionConfigs {
      * @param configLoader  Supplies DataSourceConfigurations to build the dimensions from.
      */
     public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>> configLoader) {
-        dimensionConfigs = configLoader.get().stream()
-                .flatMap(tableName -> tableName.getDimensions().stream())
-                .distinct()
-                .map(dimensionName -> new DefaultKeyValueStoreDimensionConfig(
-                        () -> dimensionName,
-                        dimensionName,
-                        "",
-                        dimensionName,
-                        "General",
-                        getDefaultFields(),
-                        getDefaultKeyValueStore(dimensionName),
-                        getDefaultSearchProvider(dimensionName)
-                ))
-                .collect(
-                        Collectors.collectingAndThen(
-                                Collectors.toSet(),
-                                Collections::unmodifiableSet
-                        )
-                );
+        dimensionConfigs = new HashMap<>();
+        configLoader.get()
+                .forEach(dataSourceConfiguration -> {
+                    Set<DimensionConfig> tableDimensionConfigs = dataSourceConfiguration.getDimensions().stream()
+                            .map(dimensionName -> new DefaultKeyValueStoreDimensionConfig(
+                                            () -> dimensionName,
+                                            dimensionName,
+                                            "",
+                                            dimensionName,
+                                            "General",
+                                            getDefaultFields(),
+                                            getDefaultKeyValueStore(dimensionName),
+                                            getDefaultSearchProvider(dimensionName)
+                                    )
+                            ).collect(
+                                    Collectors.collectingAndThen(
+                                            Collectors.toSet(),
+                                            Collections::unmodifiableSet
+                                    ));
+
+                    dimensionConfigs.put(dataSourceConfiguration.getName(), tableDimensionConfigs);
+                });
+        ;
     }
 
     /**
@@ -59,7 +65,21 @@ public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>>
      * @return set of dimension configurations
      */
     public Set<DimensionConfig> getAllDimensionConfigurations() {
-        return dimensionConfigs;
+        return dimensionConfigs.values().stream()
+                .flatMap(Set::stream)
+                .distinct()
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * Get all the dimension configurations associated with this datasource.
+     *
+     * @param dataSourceConfiguration the datasource configuration's dimensions to load
+     *
+     * @return the dimension configurations for this datasource
+     */
+    public Set<DimensionConfig> getDimensionConfigs(DataSourceConfiguration dataSourceConfiguration) {
+        return dimensionConfigs.getOrDefault(dataSourceConfiguration.getName(), Collections.emptySet());
     }
 
     /**
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/metric/GenericMetricLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/metric/GenericMetricLoader.java
index cfe314831a..4498b4360a 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/metric/GenericMetricLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/metric/GenericMetricLoader.java
@@ -40,7 +40,7 @@ public GenericMetricLoader(
     }
 
     /**
-     * (Re)Initialize the metric makers with the given metric dictionary.
+     * Initialize the metric makers with the given metric dictionary.
      *
      * @param metricDictionary  Metric dictionary to use for generating the metric makers.
      */
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
index 0e0b7b0269..801ac43fee 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
@@ -32,12 +32,12 @@
  * Load the table configuration for any druid setup.
  */
 public class GenericTableLoader extends BaseTableLoader {
-    private Map<String, Set<Granularity>> validGrains = new HashMap<>();
+    private final Map<String, Set<Granularity>> validGrains = new HashMap<>();
     // Set up the metrics
-    private Map<String, Set<FieldName>> druidMetricNames = new HashMap<>();
-    private Map<String, Set<ApiMetricName>> apiMetricNames = new HashMap<>();
+    private final Map<String, Set<FieldName>> druidMetricNames = new HashMap<>();
+    private final Map<String, Set<ApiMetricName>> apiMetricNames = new HashMap<>();
     // Set up the table definitions
-    private Map<String, Set<PhysicalTableDefinition>> tableDefinitions = new HashMap<>();
+    private final Map<String, Set<PhysicalTableDefinition>> tableDefinitions = new HashMap<>();
     private final Supplier<List<? extends DataSourceConfiguration>> configLoader;
 
     /**
@@ -69,13 +69,14 @@ private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
                     dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(DruidMetricName::new)
-                    .collect(Collectors.toSet()));
+                    .collect(Collectors.toSet())
+            );
 
             tableDefinitions.put(dataSourceConfiguration.getName(),
                     getPhysicalTableDefinitions(
                             dataSourceConfiguration,
                             dataSourceConfiguration.getValidTimeGrain(),
-                            genericDimensionConfigs.getAllDimensionConfigurations()
+                            genericDimensionConfigs.getDimensionConfigs(dataSourceConfiguration)
                     )
             );
 
diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
index 71c323367c..fb0f72b26f 100644
--- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
@@ -36,7 +36,7 @@ class ConfigurationLoaderSpec extends Specification {
         StaticWikiConfigLoader wikiConfigLoader = new StaticWikiConfigLoader();
         GenericDimensionConfigs genericDimensions = new GenericDimensionConfigs(wikiConfigLoader);
         LinkedHashSet<DimensionConfig> dimensions = genericDimensions.
-                getAllDimensionConfigurations();
+                getAllDimensionConfigurations(dataSourceConfiguration.getName());
         loader = new ConfigurationLoader(
                 new KeyValueStoreDimensionLoader(dimensions),
                 new GenericMetricLoader(wikiConfigLoader),

From 246f21ae01629908ab35100761576a66f23f6b44 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Thu, 1 Jun 2017 13:04:47 -0500
Subject: [PATCH 07/12] Fix refactoring which didn't get noticed in a test spec

---
 .../wiki/webservice/application/ConfigurationLoaderSpec.groovy  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
index fb0f72b26f..71c323367c 100644
--- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/application/ConfigurationLoaderSpec.groovy
@@ -36,7 +36,7 @@ class ConfigurationLoaderSpec extends Specification {
         StaticWikiConfigLoader wikiConfigLoader = new StaticWikiConfigLoader();
         GenericDimensionConfigs genericDimensions = new GenericDimensionConfigs(wikiConfigLoader);
         LinkedHashSet<DimensionConfig> dimensions = genericDimensions.
-                getAllDimensionConfigurations(dataSourceConfiguration.getName());
+                getAllDimensionConfigurations();
         loader = new ConfigurationLoader(
                 new KeyValueStoreDimensionLoader(dimensions),
                 new GenericMetricLoader(wikiConfigLoader),

From a5b784a42bc495f113b8d3603c6c01a0106f0bee Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Thu, 1 Jun 2017 13:51:29 -0500
Subject: [PATCH 08/12] Clarify variable names

---
 .../dimension/GenericDimensionConfigs.java    | 11 ++++----
 .../data/config/table/GenericTableLoader.java | 26 +++++++++----------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
index 612f310895..e7768c0987 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
@@ -26,7 +26,7 @@
  * Hold all the dimension configurations for a generic druid configuration.
  */
 public class GenericDimensionConfigs {
-    private final Map<String, Set<DimensionConfig>> dimensionConfigs;
+    private final Map<String, Set<DimensionConfig>> dataSourceToDimensionConfigs;
 
     /**
      * Construct the dimension configurations.
@@ -34,7 +34,7 @@ public class GenericDimensionConfigs {
      * @param configLoader  Supplies DataSourceConfigurations to build the dimensions from.
      */
     public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>> configLoader) {
-        dimensionConfigs = new HashMap<>();
+        dataSourceToDimensionConfigs = new HashMap<>();
         configLoader.get()
                 .forEach(dataSourceConfiguration -> {
                     Set<DimensionConfig> tableDimensionConfigs = dataSourceConfiguration.getDimensions().stream()
@@ -54,9 +54,8 @@ public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>>
                                             Collections::unmodifiableSet
                                     ));
 
-                    dimensionConfigs.put(dataSourceConfiguration.getName(), tableDimensionConfigs);
+                    dataSourceToDimensionConfigs.put(dataSourceConfiguration.getName(), tableDimensionConfigs);
                 });
-        ;
     }
 
     /**
@@ -65,7 +64,7 @@ public GenericDimensionConfigs(Supplier<List<? extends DataSourceConfiguration>>
      * @return set of dimension configurations
      */
     public Set<DimensionConfig> getAllDimensionConfigurations() {
-        return dimensionConfigs.values().stream()
+        return dataSourceToDimensionConfigs.values().stream()
                 .flatMap(Set::stream)
                 .distinct()
                 .collect(Collectors.toSet());
@@ -79,7 +78,7 @@ public Set<DimensionConfig> getAllDimensionConfigurations() {
      * @return the dimension configurations for this datasource
      */
     public Set<DimensionConfig> getDimensionConfigs(DataSourceConfiguration dataSourceConfiguration) {
-        return dimensionConfigs.getOrDefault(dataSourceConfiguration.getName(), Collections.emptySet());
+        return dataSourceToDimensionConfigs.getOrDefault(dataSourceConfiguration.getName(), Collections.emptySet());
     }
 
     /**
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
index 801ac43fee..f304a5b8ed 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
@@ -32,12 +32,12 @@
  * Load the table configuration for any druid setup.
  */
 public class GenericTableLoader extends BaseTableLoader {
-    private final Map<String, Set<Granularity>> validGrains = new HashMap<>();
+    private final Map<String, Set<Granularity>> dataSourceToValidGrains = new HashMap<>();
     // Set up the metrics
-    private final Map<String, Set<FieldName>> druidMetricNames = new HashMap<>();
-    private final Map<String, Set<ApiMetricName>> apiMetricNames = new HashMap<>();
+    private final Map<String, Set<FieldName>> dataSourceToDruidMetricNames = new HashMap<>();
+    private final Map<String, Set<ApiMetricName>> dataSourceToApiMetricNames = new HashMap<>();
     // Set up the table definitions
-    private final Map<String, Set<PhysicalTableDefinition>> tableDefinitions = new HashMap<>();
+    private final Map<String, Set<PhysicalTableDefinition>> dataSourceToTableDefinitions = new HashMap<>();
     private final Supplier<List<? extends DataSourceConfiguration>> configLoader;
 
     /**
@@ -65,14 +65,14 @@ public GenericTableLoader(
     private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
         configLoader.get().forEach(dataSourceConfiguration -> {
 
-            druidMetricNames.put(dataSourceConfiguration.getName(),
+            dataSourceToDruidMetricNames.put(dataSourceConfiguration.getName(),
                     dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(DruidMetricName::new)
                     .collect(Collectors.toSet())
             );
 
-            tableDefinitions.put(dataSourceConfiguration.getName(),
+            dataSourceToTableDefinitions.put(dataSourceConfiguration.getName(),
                     getPhysicalTableDefinitions(
                             dataSourceConfiguration,
                             dataSourceConfiguration.getValidTimeGrain(),
@@ -80,14 +80,14 @@ private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
                     )
             );
 
-            apiMetricNames.put(dataSourceConfiguration.getName(),
+            dataSourceToApiMetricNames.put(dataSourceConfiguration.getName(),
                     dataSourceConfiguration.getMetrics()
                     .stream()
                     .map(metricName -> new FiliApiMetricName(metricName, dataSourceConfiguration.getValidTimeGrain()))
                     .collect(Collectors.toSet())
             );
 
-            validGrains.put(dataSourceConfiguration.getName(), getGranularities(dataSourceConfiguration));
+            dataSourceToValidGrains.put(dataSourceConfiguration.getName(), getGranularities(dataSourceConfiguration));
         });
 
     }
@@ -129,7 +129,7 @@ private Set<PhysicalTableDefinition> getPhysicalTableDefinitions(
                         new ConcretePhysicalTableDefinition(
                                 dataSourceConfiguration.getTableName(),
                                 zonedTimeGrain,
-                                druidMetricNames.get(dataSourceConfiguration.getName()),
+                                dataSourceToDruidMetricNames.get(dataSourceConfiguration.getName()),
                                 dimsBasefactDruidTable
                         )
                 )
@@ -145,22 +145,22 @@ private Set<PhysicalTableDefinition> getPhysicalTableDefinitions(
     public void loadTableDictionary(ResourceDictionaries dictionaries) {
         configLoader.get()
                 .forEach(table -> {
-                    Set<TableName> currentTableGroupTableNames = tableDefinitions.get(table.getName())
+                    Set<TableName> currentTableGroupTableNames = dataSourceToTableDefinitions.get(table.getName())
                             .stream()
                             .map(PhysicalTableDefinition::getName)
                             .collect(Collectors.toSet());
 
                     TableGroup tableGroup = buildDimensionSpanningTableGroup(
                             currentTableGroupTableNames,
-                            tableDefinitions.get(table.getName()),
+                            dataSourceToTableDefinitions.get(table.getName()),
                             dictionaries,
-                            apiMetricNames.get(table.getName())
+                            dataSourceToApiMetricNames.get(table.getName())
                     );
 
                     loadLogicalTableWithGranularities(
                             table.getTableName().asName(),
                             tableGroup,
-                            validGrains.get(table.getName()),
+                            dataSourceToValidGrains.get(table.getName()),
                             dictionaries
                     );
                 });

From 5a24c311b755f705cd18aad77e0e650c2809d80e Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Thu, 1 Jun 2017 16:17:11 -0500
Subject: [PATCH 09/12] Address Jiaqi's comments, add changelog, fix readme

- The druid dimension loader is now causing it to fail because the
tables can not be found in the DataSourceMetadataService
- Enabled DruidDimensionLoader
---
 CHANGELOG.md                                  |  4 ++
 fili-generic-example/README.md                |  5 +-
 .../dimension/GenericDimensionConfigs.java    |  4 +-
 .../data/config/table/GenericTableLoader.java | 54 ++++++++++++++-----
 .../resources/applicationConfig.properties    |  1 +
 .../AutomaticDruidConfigLoaderSpec.groovy     |  1 -
 ...ipleDatasourceDruidConfigLoaderSpec.groovy | 52 +++++++++---------
 7 files changed, 77 insertions(+), 44 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b17ad439d..ef7b854b7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -500,6 +500,10 @@ Removals:
 
 ### Fixed:
 
+- [Fix the generic example for loading multiple tables](https://github.com/yahoo/fili/pull/309)
+    * Loading multiple tables caused it to hang and eventually time out.
+    * Also fixed issue causing all tables to show the same set of dimensions.
+
 - [Support for Lucene 5 indexes restored](https://github.com/yahoo/fili/pull/265)
     * Added `lucene-backward-codecs.jar` as a dependency to restore support for indexes built on earlier instances.
 
diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md
index 3dbc0f7a77..c031643f9f 100644
--- a/fili-generic-example/README.md
+++ b/fili-generic-example/README.md
@@ -25,7 +25,10 @@ mvn -pl fili-generic-example exec:java
 - Note that if your setup is different you can adjust it by changing the default parameters below
 
     ```bash
-    mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1
+    mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 \
+    -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 \
+    -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 \
+    -Dbard__ui_druid_broker=http://localhost:8082/druid/v2
     ```
 
 From another window, run a test query against the default druid data.
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
index e7768c0987..b7170fd4ea 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/dimension/GenericDimensionConfigs.java
@@ -73,7 +73,7 @@ public Set<DimensionConfig> getAllDimensionConfigurations() {
     /**
      * Get all the dimension configurations associated with this datasource.
      *
-     * @param dataSourceConfiguration the datasource configuration's dimensions to load
+     * @param dataSourceConfiguration  The datasource configuration's dimensions to load
      *
      * @return the dimension configurations for this datasource
      */
@@ -84,7 +84,7 @@ public Set<DimensionConfig> getDimensionConfigs(DataSourceConfiguration dataSour
     /**
      * Lazily provide a KeyValueStore for this store name.
      *
-     * @param storeName  the name for the key value store
+     * @param storeName  The name for the key value store
      *
      * @return A KeyValueStore instance
      */
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
index f304a5b8ed..87d7e27651 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/table/GenericTableLoader.java
@@ -5,6 +5,7 @@
 import com.yahoo.bard.webservice.data.config.ResourceDictionaries;
 import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig;
 import com.yahoo.bard.webservice.data.config.names.ApiMetricName;
+import com.yahoo.bard.webservice.data.config.names.DataSourceName;
 import com.yahoo.bard.webservice.data.config.names.FieldName;
 import com.yahoo.bard.webservice.data.config.names.TableName;
 import com.yahoo.bard.webservice.data.config.table.BaseTableLoader;
@@ -15,19 +16,27 @@
 import com.yahoo.bard.webservice.data.time.ZonelessTimeGrain;
 import com.yahoo.bard.webservice.druid.model.query.AllGranularity;
 import com.yahoo.bard.webservice.druid.model.query.Granularity;
+import com.yahoo.bard.webservice.metadata.DataSourceMetadata;
 import com.yahoo.bard.webservice.metadata.DataSourceMetadataService;
 import com.yahoo.bard.webservice.table.TableGroup;
 import com.yahoo.wiki.webservice.data.config.auto.DataSourceConfiguration;
 import com.yahoo.wiki.webservice.data.config.dimension.GenericDimensionConfigs;
 import com.yahoo.wiki.webservice.data.config.metric.DruidMetricName;
 import com.yahoo.wiki.webservice.data.config.metric.FiliApiMetricName;
+
 import org.joda.time.DateTimeZone;
 
-import javax.validation.constraints.NotNull;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import javax.validation.constraints.NotNull;
+
 /**
  * Load the table configuration for any druid setup.
  */
@@ -54,25 +63,40 @@ public GenericTableLoader(
     ) {
         super(metadataService);
         this.configLoader = configLoader;
-        configureTables(genericDimensionConfigs);
+        configureTables(genericDimensionConfigs, metadataService);
     }
 
     /**
      * Set up the tables for this table loader.
      *
      * @param genericDimensionConfigs  The dimensions to load into test tables.
+     * @param metadataService  The metadata service to plays the datasources in.
      */
-    private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
+    private void configureTables(
+            GenericDimensionConfigs genericDimensionConfigs,
+            DataSourceMetadataService metadataService
+    ) {
         configLoader.get().forEach(dataSourceConfiguration -> {
 
-            dataSourceToDruidMetricNames.put(dataSourceConfiguration.getName(),
+            metadataService.update(
+                    DataSourceName.of(dataSourceConfiguration.getName()),
+                    new DataSourceMetadata(
+                            dataSourceConfiguration.getName(),
+                            Collections.emptyMap(),
+                            Collections.emptyList()
+                    )
+            );
+
+            dataSourceToDruidMetricNames.put(
+                    dataSourceConfiguration.getName(),
                     dataSourceConfiguration.getMetrics()
-                    .stream()
-                    .map(DruidMetricName::new)
-                    .collect(Collectors.toSet())
+                            .stream()
+                            .map(DruidMetricName::new)
+                            .collect(Collectors.toSet())
             );
 
-            dataSourceToTableDefinitions.put(dataSourceConfiguration.getName(),
+            dataSourceToTableDefinitions.put(
+                    dataSourceConfiguration.getName(),
                     getPhysicalTableDefinitions(
                             dataSourceConfiguration,
                             dataSourceConfiguration.getValidTimeGrain(),
@@ -80,11 +104,15 @@ private void configureTables(GenericDimensionConfigs genericDimensionConfigs) {
                     )
             );
 
-            dataSourceToApiMetricNames.put(dataSourceConfiguration.getName(),
+            dataSourceToApiMetricNames.put(
+                    dataSourceConfiguration.getName(),
                     dataSourceConfiguration.getMetrics()
-                    .stream()
-                    .map(metricName -> new FiliApiMetricName(metricName, dataSourceConfiguration.getValidTimeGrain()))
-                    .collect(Collectors.toSet())
+                            .stream()
+                            .map(metricName -> new FiliApiMetricName(
+                                    metricName,
+                                    dataSourceConfiguration.getValidTimeGrain()
+                            ))
+                            .collect(Collectors.toSet())
             );
 
             dataSourceToValidGrains.put(dataSourceConfiguration.getName(), getGranularities(dataSourceConfiguration));
diff --git a/fili-generic-example/src/main/resources/applicationConfig.properties b/fili-generic-example/src/main/resources/applicationConfig.properties
index e40871c711..c5130a8c1e 100644
--- a/fili-generic-example/src/main/resources/applicationConfig.properties
+++ b/fili-generic-example/src/main/resources/applicationConfig.properties
@@ -13,6 +13,7 @@ bard__non_ui_druid_broker=http://localhost:8082/druid/v2
 bard__ui_druid_broker=http://localhost:8082/druid/v2
 bard__druid_coord=http://localhost:8081/druid/coordinator/v1
 bard__fili_port=9998
+bard__druid_dimensions_loader_enabled = true
 
 # Use memory for the default dimension backing store
 bard__dimension_backend=memory
diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
index 0ed2d72afd..1412964c6f 100644
--- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/AutomaticDruidConfigLoaderSpec.groovy
@@ -52,7 +52,6 @@ public class AutomaticDruidConfigLoaderSpec extends Specification {
             } else if (druidWebService.lastUrl == '/datasources/' + datasource + "/?full") {
                 return expectedMetricsAndDimensions
             }
-            println "No response for " + druidWebService.lastUrl
         }
     }
 
diff --git a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy
index 86df7170b5..53f41eec05 100644
--- a/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy
+++ b/fili-generic-example/src/test/groovy/com/yahoo/wiki/webservice/web/endpoints/MultipleDatasourceDruidConfigLoaderSpec.groovy
@@ -25,7 +25,6 @@ class MultipleDatasourceDruidConfigLoaderSpec extends Specification {
             } else if (druidWebService.lastUrl.equals("/datasources/table2/?full")) {
                 return getFullTable(datasources[1], table2_metrics, table2_dimensions)
             }
-            println "No response for " + druidWebService.lastUrl
         }
     }
 
@@ -50,31 +49,30 @@ class MultipleDatasourceDruidConfigLoaderSpec extends Specification {
 
     private static String getFullTable(String name, String[] metrics, String[] dimensions) {
         return """{
-    "name": "${name}",
-    "properties": {},
-    "segments": [
-        {
-            "dataSource": "${name}",
-            "interval": "2015-09-12T00:00:00.000Z/2015-09-13T00:00:00.000Z",
-            "version": "2017-02-27T03:06:09.422Z",
-            "loadSpec": 
-                {
-                    "type": "local",
-                    "path": "2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z/2017-02-27T03:06:09.422Z/0/index.zip"
-                },
-            "dimensions": "${dimensions.join(",")}",
-            "metrics":  "${metrics.join(",")}",
-            "shardSpec": 
-                {
-                    "type": "none"
-                },
-            "binaryVersion": 9,
-            "size": 5537610,
-            "identifier": "${name}_2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z_2017-02-27T03:06:09.422Z"
-        }
-    ]
-}
-
-"""
+                    "name": "${name}",
+                    "properties": {},
+                    "segments": [
+                        {
+                            "dataSource": "${name}",
+                            "interval": "2015-09-12T00:00:00.000Z/2015-09-13T00:00:00.000Z",
+                            "version": "2017-02-27T03:06:09.422Z",
+                            "loadSpec": 
+                                {
+                                    "type": "local",
+                                    "path": "2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z/2017-02-27T03:06:09.422Z/0/index.zip"
+                                },
+                            "dimensions": "${dimensions.join(",")}",
+                            "metrics":  "${metrics.join(",")}",
+                            "shardSpec": 
+                                {
+                                    "type": "none"
+                                },
+                            "binaryVersion": 9,
+                            "size": 5537610,
+                            "identifier": "${name}_2015-09-12T00:00:00.000Z_2015-09-13T00:00:00.000Z_2017-02-27T03:06:09.422Z"
+                        }
+                    ]
+                }
+                """
     }
 }

From 76b130cbab0d58b6147b94295a43dbd969b5ea40 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Fri, 2 Jun 2017 09:58:59 -0500
Subject: [PATCH 10/12] Clarify instructions in the readme

---
 fili-generic-example/README.md | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md
index c031643f9f..025b09a248 100644
--- a/fili-generic-example/README.md
+++ b/fili-generic-example/README.md
@@ -15,12 +15,11 @@ In order to set up, this will connect to druid at  [http://localhost:8081/druid/
     ```
 3. Use Maven to install and launch the Fili Generic example:
 
-
-```bash
-cd fili
-mvn install
-mvn -pl fili-generic-example exec:java
-```
+    ```bash
+    cd fili
+    mvn install
+    mvn -pl fili-generic-example exec:java
+    ```
 
 - Note that if your setup is different you can adjust it by changing the default parameters below
 
@@ -39,21 +38,21 @@ Here are some sample queries that you can run to verify your server:
 
 ### Any Server
 
-- List tables:
+- List [tables](http://localhost:9998/v1/tables):
   
       GET http://localhost:9998/v1/tables
 
-- List dimensions:  
+- List [dimensions](http://localhost:9998/v1/dimensions):  
 
       GET http://localhost:9998/v1/dimensions
 
-- List metrics:
+- List [metrics](http://localhost:9998/v1/metrics/):
   
       GET http://localhost:9998/v1/metrics/
 
 ### Specific to Wikipedia data
 
-- If everything is working, the query below
+- If everything is working, the [query below](http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H)
     ```bash
     curl "http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H" -H "Content-Type: application/json" | python -m json.tool
     ```
@@ -70,7 +69,9 @@ Here are some sample queries that you can run to verify your server:
 - Count of edits by hour for the last 72 hours:  
   
       GET http://localhost:9998/v1/data/wikiticker/day/?metrics=count&dateTime=PT72H/current
-    Note: this will should be something like the response below unless you have streaming data.
+    
+    Note: this will should be something like the response below since the 
+    wikiticker table doesn't have data for the past 72 hours from now.
     ```json
     {
         "rows": [],
@@ -80,7 +81,8 @@ Here are some sample queries that you can run to verify your server:
     }
     ```  
 
-- Show debug info, including the query sent to Druid:  
+- Show [debug info](http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current),
+ including the query sent to Druid:  
 
       GET http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current
 
@@ -93,4 +95,6 @@ Here are some sample queries that you can run to verify your server:
 
 1. In IntelliJ, go to `File -> Open`
 2. Select the `pom.xml` file at the root of the project
-3. Run `GenericMain` which can be found in `fili-generic-example` (e.g. right click and choose run)
+3. Under `src/main/resources/applicationConfig.properties`, change `bard__non_ui_druid_broker`, 
+`bard__ui_druid_broker`, `bard__druid_coord`, and other properties as needed.
+4. Run `GenericMain` which can be found in `fili-generic-example` (e.g. right click and choose run)

From 0780803e7e1c290229f2418938d8ed03cfee914a Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Thu, 8 Jun 2017 08:26:35 -0500
Subject: [PATCH 11/12] Address Rick's comments and clarify readme for
 Wikipedia Example

---
 fili-generic-example/README.md                             | 7 ++++++-
 .../wiki/webservice/data/config/auto/DruidNavigator.java   | 6 ++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md
index 025b09a248..ef5691d0d3 100644
--- a/fili-generic-example/README.md
+++ b/fili-generic-example/README.md
@@ -94,7 +94,12 @@ Here are some sample queries that you can run to verify your server:
 ## Importing and Running in IntelliJ
 
 1. In IntelliJ, go to `File -> Open`
+
 2. Select the `pom.xml` file at the root of the project
+    
+    **NOTE:** if you're running this locally and haven't changed any settings (like the Wikipedia example) 
+    you can **skip step 3**.
 3. Under `src/main/resources/applicationConfig.properties`, change `bard__non_ui_druid_broker`, 
-`bard__ui_druid_broker`, `bard__druid_coord`, and other properties as needed.
+`bard__ui_druid_broker`, `bard__druid_coord`, and other properties.
+    
 4. Run `GenericMain` which can be found in `fili-generic-example` (e.g. right click and choose run)
diff --git a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
index 147d4d705a..e47f3ef633 100644
--- a/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
+++ b/fili-generic-example/src/main/java/com/yahoo/wiki/webservice/data/config/auto/DruidNavigator.java
@@ -2,12 +2,14 @@
 // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
 package com.yahoo.wiki.webservice.data.config.auto;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.bard.webservice.data.time.DefaultTimeGrain;
 import com.yahoo.bard.webservice.data.time.TimeGrain;
 import com.yahoo.bard.webservice.druid.client.DruidWebService;
 import com.yahoo.bard.webservice.druid.client.SuccessCallback;
 import com.yahoo.bard.webservice.util.IntervalUtils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
 import org.asynchttpclient.Response;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -67,7 +69,7 @@ public List<? extends DataSourceConfiguration> get() {
      * ["wikiticker"]
      */
     private void loadAllDatasources() {
-        final List<Future<Response>> fullTableResponses = new ArrayList<>();
+        List<Future<Response>> fullTableResponses = new ArrayList<>();
         Future<Response> responseFuture = queryDruid(rootNode -> {
             if (rootNode.isArray()) {
                 rootNode.forEach(jsonNode -> {

From b6d2420edf6aed59ec2e130ca4f1b6cca6cf6cc9 Mon Sep 17 00:00:00 2001
From: Kevin Hinterlong <kevinhinterlong@gmail.com>
Date: Fri, 9 Jun 2017 13:34:51 -0500
Subject: [PATCH 12/12] Add a note in README for last working version

- Also fix list of required settings to top of readme
---
 fili-generic-example/README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md
index ef5691d0d3..c414199792 100644
--- a/fili-generic-example/README.md
+++ b/fili-generic-example/README.md
@@ -1,9 +1,15 @@
 Fili Generic Loader Application
 ==================================
 
-This application will automatically configure fili to work with **any** instance of Druid and show the basic metrics and dimensions. This lets you test what it's like using Fili without putting any effort into setting it up.
+This application will automatically configure fili to work with **any** instance
+ of Druid and show the basic metrics and dimensions. This lets you test what it's
+  like using Fili without putting any effort into setting it up.
 
-In order to set up, this will connect to druid at  [http://localhost:8081/druid/coordinator/v1](http://localhost:8081/druid/coordinator/v1). If your set up is different, you'll have to change the `bard__druid_coord` url in `applicationConfig.properties`.
+In order to set up, this will connect to druid at  [http://localhost:8081/druid/coordinator/v1](http://localhost:8081/druid/coordinator/v1).
+ If your set up is different, you'll have to change the `bard__druid_coord`,
+  `bard__non_ui_druid_broker`, `bard__ui_druid_broker` url in `applicationConfig.properties`.
+  
+Note that this was last tested using [version 0.9.1](https://github.com/yahoo/fili/tree/0.9.1)
 
 ## Setup and Launching