Config slurper master

fili-core/src/main/java/com/yahoo/bard/webservice/application/DruidDimensionsLoader.java

@@ -74,6 +74,7 @@ public class DruidDimensionsLoader extends Loader<Boolean> {
 
     /**
      * DruidDimensionsLoader fetches data from Druid and adds it to the dimension cache.
+     * The dimensions loaded are taken from the system config.
      *
      * @param physicalTableDictionary  The physical tables
      * @param dimensionDictionary  The dimensions to update
@@ -83,6 +84,30 @@ public DruidDimensionsLoader(
             PhysicalTableDictionary physicalTableDictionary,
             DimensionDictionary dimensionDictionary,
             DruidWebService druidWebService
+    ) {
+        this(
+                physicalTableDictionary,
+                dimensionDictionary,
+                //Our configuration framework automatically converts a comma-separated config value into a list.
+                SYSTEM_CONFIG.getListProperty(DRUID_DIM_LOADER_DIMENSIONS),
+                druidWebService
+        );
+    }
+
+    /**
+     * DruidDimensionsLoader fetches data from Druid and adds it to the dimension cache.
+     * The dimensions to be loaded can be passed in as a parameter.
+     *
+     * @param physicalTableDictionary  The physical tables
+     * @param dimensionDictionary  The dimension dictionary to load dimensions from.
+     * @param dimensionsToLoad  The dimensions to use.
+     * @param druidWebService  The druid webservice to query.
+     */
+    public DruidDimensionsLoader(
+            PhysicalTableDictionary physicalTableDictionary,
+            DimensionDictionary dimensionDictionary,
+            List<String> dimensionsToLoad,
+            DruidWebService druidWebService
     ) {
         super(
                 DruidDimensionsLoader.class.getSimpleName(),
@@ -100,17 +125,14 @@ public DruidDimensionsLoader(
 
         lastRunTimestamp = new AtomicReference<>();
 
-        //Our configuration framework automatically converts a comma-separated config value into a list.
-        List<String> dimensionStr = SYSTEM_CONFIG.getListProperty(DRUID_DIM_LOADER_DIMENSIONS);
-
-       // A DruidSearchQuery requires a list of dimensions, which we would have to explicitly create at serialization
-       // time if `dimensions` were a flat list instead of a list of singleton lists.
-        dimensions = dimensionStr.stream()
+        // A DruidSearchQuery requires a list of dimensions, which we would have to explicitly create at serialization
+        // time if `dimensions` were a flat list instead of a list of singleton lists.
+        this.dimensions = dimensionsToLoad.stream()
                 .map(dimensionDictionary::findByApiName)
                 .map(Collections::singletonList)
                 .collect(Collectors.toList());
 
-        dataSources = physicalTableDictionary.values().stream()
+        this.dataSources = physicalTableDictionary.values().stream()
                 .filter(physicalTable -> physicalTable instanceof ConcretePhysicalTable)
                 .map(physicalTable -> (ConcretePhysicalTable) physicalTable)
                 .map(TableDataSource::new)

fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/DefaultDimensionField.java

@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc.
+// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
+package com.yahoo.bard.webservice.data.config.dimension;
+
+import com.yahoo.bard.webservice.data.dimension.DimensionField;
+
+/**
+ * DimensionField enum.
+ */
+public enum DefaultDimensionField implements DimensionField {
+    ID("Dimension ID");
+
+    private String description;
+    private String camelName;
+
+    /**
+     * Constructor.
+     *
+     * @param description  Description of this field
+     */
+    DefaultDimensionField(String description) {
+        this.description = description;
+        this.camelName = name();
+    }
+
+    @Override
+    public String getName() {
+        return camelName;
+    }
+
+    @Override
+    public String getDescription() {
+        return description;
+    }
+
+    @Override
+    public String toString() {
+        return camelName;
+    }
+}

fili-core/src/main/java/com/yahoo/bard/webservice/util/IntervalUtils.java

@@ -5,6 +5,8 @@
 import com.yahoo.bard.webservice.config.SystemConfig;
 import com.yahoo.bard.webservice.config.SystemConfigProvider;
 import com.yahoo.bard.webservice.data.time.ZonedTimeGrain;
+import com.yahoo.bard.webservice.data.time.StandardGranularityParser;
+import com.yahoo.bard.webservice.data.time.TimeGrain;
 import com.yahoo.bard.webservice.druid.model.query.Granularity;
 import com.yahoo.bard.webservice.table.PhysicalTable;
 import com.yahoo.bard.webservice.table.PhysicalTableSchema;
@@ -230,4 +232,21 @@ public static Optional<ZonedTimeGrain> getCoarsestTimeGrain(Collection<PhysicalT
                 .map(PhysicalTable::getSchema)
                 .map(PhysicalTableSchema::getTimeGrain);
     }
+
+    /**
+     * Find a valid timegrain for the interval based on the start and end date of the interval.
+     *
+     * @param interval  The interval to find a timegrain for.
+     * @return the valid timegrain spanned by the interval.
+     */
+    public static Optional<TimeGrain> getTimeGrain(Interval interval) {
+        return StandardGranularityParser.getDefaultGrainMap()
+                .values()
+                .stream()
+                .filter(granularity -> granularity instanceof TimeGrain)
+                .map(granularity -> (TimeGrain) granularity)
+                .filter(timeGrain -> timeGrain.aligns(interval))
+                .filter(timeGrain -> interval.getStart().plus(timeGrain.getPeriod()).equals(interval.getEnd()))
+                .findFirst();
+    }
 }

fili-core/src/test/groovy/com/yahoo/bard/webservice/util/IntervalUtilsSpec.groovy

@@ -10,6 +10,8 @@ import static com.yahoo.bard.webservice.data.time.DefaultTimeGrain.QUARTER
 import static com.yahoo.bard.webservice.data.time.DefaultTimeGrain.WEEK
 import static com.yahoo.bard.webservice.data.time.DefaultTimeGrain.YEAR
 
+import com.yahoo.bard.webservice.data.time.DefaultTimeGrain
+import com.yahoo.bard.webservice.data.time.TimeGrain
 import com.yahoo.bard.webservice.data.time.ZonedTimeGrain
 import com.yahoo.bard.webservice.druid.model.query.AllGranularity
 import com.yahoo.bard.webservice.druid.model.query.Granularity
@@ -377,6 +379,31 @@ class IntervalUtilsSpec extends Specification {
         IntervalUtils.getCoarsestTimeGrain([]) == Optional.empty()
     }
 
+    @Unroll
+    def "test getTimeGrain from interval - #expectedTimeGrain"() {
+        setup:
+        String[] dates = stringInterval.split("/")
+        DateTime start = new DateTime(dates[0], DateTimeZone.UTC)
+        DateTime end = new DateTime(dates[1], DateTimeZone.UTC)
+        Interval interval = new Interval(start.toInstant(), end.toInstant())
+
+        when: "we parse the time"
+        Optional<TimeGrain> timeGrain = IntervalUtils.getTimeGrain(interval)
+
+        then: "what we expect"
+        timeGrain == expectedTimeGrain
+
+        where:
+        stringInterval                                      | expectedTimeGrain
+        "2015-01-01T00:00:00.000Z/2016-01-01T00:00:00.000Z" | Optional.of(YEAR)
+        "2017-01-01T00:00:00.000Z/2017-02-01T00:00:00.000Z" | Optional.of(MONTH)
+        "2017-02-27T00:00:00.000Z/2017-03-06T00:00:00.000Z" | Optional.of(WEEK)
+        "2015-09-12T00:00:00.000Z/2015-09-13T01:01:01.000Z" | Optional.empty()
+        "2015-09-12T00:00:00.000Z/2015-09-13T00:00:00.000Z" | Optional.of(DAY)
+        "2015-09-12T00:00:00.000Z/2015-09-12T01:00:00.000Z" | Optional.of(DefaultTimeGrain.HOUR)
+        "2015-09-12T00:00:00.000Z/2015-09-12T00:01:00.000Z" | Optional.of(DefaultTimeGrain.MINUTE)
+    }
+
     /**
      * Returns the instant at which this year began.
      *

fili-generic-example/README.md

@@ -0,0 +1,93 @@
+Fili Generic Loader Application
+==================================
+
+This application will automatically configure fili to work with **any** instance of Druid and show the basic metrics and dimensions. This lets you test what it's like using Fili without putting any effort into setting it up.
+
+In order to set up, this will connect to druid at  [http://localhost:8081/druid/coordinator/v1](http://localhost:8081/druid/coordinator/v1). If your set up is different, you'll have to change the `bard__druid_coord` url in `applicationConfig.properties`.
+
+## Setup and Launching
+
+1. Have a [Druid](http://druid.io/docs/latest/tutorials/quickstart.html) cluster running on your Unix based machine.
+
+2. Clone this repository to your computer.
+    ```bash
+    git clone git@github.com:yahoo/fili.git
+    ```
+3. Use Maven to install and launch the Fili Generic example:
+
+
+```bash
+cd fili
+mvn install
+mvn -pl fili-generic-example exec:java
+```
+
+- Note that if your setup is different you can adjust it by changing the default parameters below
+
+    ```bash
+    mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1
+    ```
+
+From another window, run a test query against the default druid data.
+
+## Example Queries
+
+Here are some sample queries that you can run to verify your server:
+
+### Any Server
+
+- List tables:
+
+      GET http://localhost:9998/v1/tables
+
+- List dimensions:  
+
+      GET http://localhost:9998/v1/dimensions
+
+- List metrics:
+
+      GET http://localhost:9998/v1/metrics/
+
+### Specific to Wikipedia data
+
+- If everything is working, the query below
+    ```bash
+    curl "http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H" -H "Content-Type: application/json" | python -m json.tool
+    ```
+     should show something like:
+    ```
+    {
+        "rows": [{
+            "dateTime": "2015-09-12 00:00:00.000",
+            "deleted": 394298.0
+        }]
+    }
+    ```
+
+- Count of edits by hour for the last 72 hours:  
+
+      GET http://localhost:9998/v1/data/wikiticker/day/?metrics=count&dateTime=PT72H/current
+    Note: this will should be something like the response below unless you have streaming data.
+    ```json
+    {
+        "rows": [],
+        "meta": {
+            "missingIntervals": ["2017-03-30 00:00:00.000/2017-04-02 00:00:00.000"]
+        }
+    }
+    ```  
+
+- Show debug info, including the query sent to Druid:  
+
+      GET http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current
+
+## Notable Restrictions
+
+- Using this is great for testing out fili and druid, but it can't do interesting things with metrics.
+- This can only use 1 timegrain even though a datasource in druid *could* have more.
+
+## Importing and Running in IntelliJ
+
+1. In IntelliJ, go to `File -> Open`
+2. Select the `pom.xml` file at the root of the project
+3. Run `GenericMain` which can be found in `fili-generic-example` (e.g. right click and choose run)