Add extraction function to selector dimensional filter to allow lookup on filter

CHANGELOG.md

@@ -10,6 +10,9 @@ Current
 
 ### Added:
 
+- [Extraction Function on selector filter](https://github.com/yahoo/fili/pull/617)
+    * Added extraction function on dimensional filter, defaults to extraction function on dimension if it exists.
+
 - [Implement TimeFormatExtractionFunction](https://github.com/yahoo/fili/pull/611)
     * Enable [`TimeFormatExtractionFunction`](http://druid.io/docs/0.10.1/querying/dimensionspecs.html#time-format-extraction-function)
       in Fili so that API users could interact with Druid using `TimeFormatExtractionFunction` through Fili.
@@ -239,6 +242,9 @@ Current
 
 ### Deprecated:
 
+- [Extraction Function on selector filter](https://github.com/yahoo/fili/pull/617)
+    * Deprecated `ExtractionFilter` since it is deprecated in druid, use selector filter with extraction function instead
+
 - [Rename filter variables and methods in DataApiRequest](https://github.com/yahoo/fili/pull/507)
     * Deprecated `getFilters` in favor of `getApiFilters` and `getFilter` in favor of `getDruidFilter`
 

fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/DimensionalFilter.java

@@ -3,37 +3,71 @@
 package com.yahoo.bard.webservice.druid.model.filter;
 
 import com.yahoo.bard.webservice.data.dimension.Dimension;
+import com.yahoo.bard.webservice.data.dimension.impl.ExtractionFunctionDimension;
+import com.yahoo.bard.webservice.druid.model.dimension.extractionfunction.ExtractionFunction;
 import com.yahoo.bard.webservice.druid.serializers.DimensionToNameSerializer;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Filter for matching a dimension.
  *
  * @param <T> a DimensionalFilter
  */
+@JsonInclude(JsonInclude.Include.NON_NULL)
 public abstract class DimensionalFilter<T extends DimensionalFilter<? super T>> extends Filter {
 
     private final Dimension dimension;
+    private final ExtractionFunction extractionFunction;
 
     /**
-     * Constructor.
+     * Constructor, default extraction function to the one on dimension if it has one.
      *
      * @param dimension  Dimension to filter
      * @param type Type of the filter
      */
     protected DimensionalFilter(Dimension dimension, FilterType type) {
         super(type);
+
         this.dimension = dimension;
+
+        if (dimension instanceof ExtractionFunctionDimension) {
+            Optional<ExtractionFunction> optionalExtractionFunction = ((ExtractionFunctionDimension) dimension)
+                    .getExtractionFunction();
+            this.extractionFunction = optionalExtractionFunction.isPresent() ? optionalExtractionFunction.get() : null;
+        } else {
+            this.extractionFunction = null;
+        }
+    }
+
+    /**
+     * Constructor, with explicit extraction function provided.
+     *
+     * @param dimension  Dimension to filter
+     * @param type Type of the filter
+     * @param extractionFunction  Extraction function to be applied on dimension
+     */
+    protected DimensionalFilter(Dimension dimension, FilterType type, ExtractionFunction extractionFunction) {
+        super(type);
+        this.dimension = dimension;
+        this.extractionFunction = extractionFunction;
     }
 
     @JsonSerialize(using = DimensionToNameSerializer.class)
     public Dimension getDimension() {
         return dimension;
     }
 
+    @JsonProperty(value = "extractionFn")
+    public ExtractionFunction getExtractionFunction() {
+        return extractionFunction;
+    }
+
     /**
      * Get a new instance of this filter with the given Dimension.
      *
@@ -45,7 +79,7 @@ public Dimension getDimension() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), dimension);
+        return Objects.hash(super.hashCode(), dimension, extractionFunction);
     }
 
     @Override
@@ -56,6 +90,7 @@ public boolean equals(Object obj) {
 
         return
                 super.equals(obj) &&
-                Objects.equals(dimension, other.dimension);
+                        Objects.equals(dimension, other.dimension) &&
+                        Objects.equals(extractionFunction, other.extractionFunction);
     }
 }

fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/ExtractionFilter.java

@@ -5,19 +5,18 @@
 import com.yahoo.bard.webservice.data.dimension.Dimension;
 import com.yahoo.bard.webservice.druid.model.dimension.extractionfunction.ExtractionFunction;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
-
 import java.util.Objects;
 
 /**
  * Filter for matching a dimension using some specific Extraction function.
+ *
+ * @deprecated  Use {@link SelectorFilter} dimensional filters with extractionFn specified instead
  */
+@Deprecated
 public class ExtractionFilter extends DimensionalFilter {
 
     private final String value;
 
-    private final ExtractionFunction extractionFunction;
-
     /**
      * Constructor.
      *
@@ -26,23 +25,17 @@ public class ExtractionFilter extends DimensionalFilter {
      * @param extractionFn  Function to do the extraction
      */
     public ExtractionFilter(Dimension dimension, String value, ExtractionFunction extractionFn) {
-        super(dimension, DefaultFilterType.EXTRACTION);
+        super(dimension, DefaultFilterType.EXTRACTION, extractionFn);
         this.value = value;
-        this.extractionFunction = extractionFn;
     }
 
     public String getValue() {
         return value;
     }
 
-    @JsonProperty(value = "extractionFn")
-    public ExtractionFunction getExtractionFunction() {
-        return extractionFunction;
-    }
-
     @Override
     public ExtractionFilter withDimension(Dimension dimension) {
-        return new ExtractionFilter(dimension, value, extractionFunction);
+        return new ExtractionFilter(dimension, value, getExtractionFunction());
     }
 
     /**
@@ -53,7 +46,7 @@ public ExtractionFilter withDimension(Dimension dimension) {
      * @return a new instance of this filter with the given value
      */
     public ExtractionFilter withValue(String value) {
-        return new ExtractionFilter(getDimension(), value, extractionFunction);
+        return new ExtractionFilter(getDimension(), value, getExtractionFunction());
     }
 
     /**
@@ -69,7 +62,7 @@ public ExtractionFilter withExtractionFunction(ExtractionFunction extractionFunc
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), value, extractionFunction);
+        return Objects.hash(super.hashCode(), value);
     }
 
     @Override
@@ -80,7 +73,6 @@ public boolean equals(Object obj) {
 
         return
                 super.equals(obj) &&
-                Objects.equals(value, other.value) &&
-                Objects.equals(extractionFunction, other.extractionFunction);
+                Objects.equals(value, other.value);
     }
 }

fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/SelectorFilter.java

@@ -3,6 +3,7 @@
 package com.yahoo.bard.webservice.druid.model.filter;
 
 import com.yahoo.bard.webservice.data.dimension.Dimension;
+import com.yahoo.bard.webservice.druid.model.dimension.extractionfunction.ExtractionFunction;
 
 import java.util.Objects;
 
@@ -24,13 +25,25 @@ public SelectorFilter(Dimension dimension, String value) {
         this.value = value;
     }
 
+    /**
+     * Constructor.
+     *
+     * @param dimension  Dimension to apply the extraction to
+     * @param value  Value of the filter
+     * @param extractionFn  Extraction function to be applied on dimension
+     */
+    public SelectorFilter(Dimension dimension, String value, ExtractionFunction extractionFn) {
+        super(dimension, DefaultFilterType.SELECTOR, extractionFn);
+        this.value = value;
+    }
+
     public String getValue() {
         return value;
     }
 
     @Override
     public SelectorFilter withDimension(Dimension dimension) {
-        return new SelectorFilter(dimension,  value);
+        return new SelectorFilter(dimension,  value, getExtractionFunction());
     }
 
     /**
@@ -41,7 +54,18 @@ public SelectorFilter withDimension(Dimension dimension) {
      * @return a new instance of this filter with the given value
      */
     public SelectorFilter withValue(String value) {
-        return new SelectorFilter(getDimension(), value);
+        return new SelectorFilter(getDimension(), value, getExtractionFunction());
+    }
+
+    /**
+     * Get a new instance of this filter with the given value.
+     *
+     * @param extractionFn  Extraction function to be applied on dimension
+     *
+     * @return a new instance of this filter with the given value
+     */
+    public SelectorFilter withExtractionFn(ExtractionFunction extractionFn) {
+        return new SelectorFilter(getDimension(), value, extractionFn);
     }
 
     @Override

fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/model/filter/SelectorFilterSpec.groovy

@@ -0,0 +1,101 @@
+// Copyright 2017 Yahoo Inc.
+// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
+package com.yahoo.bard.webservice.druid.model.filter
+
+import com.yahoo.bard.webservice.data.dimension.impl.LookupDimension
+import com.yahoo.bard.webservice.druid.model.datasource.TableDataSource
+import com.yahoo.bard.webservice.druid.model.dimension.extractionfunction.ExtractionFunction
+import com.yahoo.bard.webservice.druid.model.dimension.extractionfunction.RegisteredLookupExtractionFunction
+import com.yahoo.bard.webservice.druid.model.query.DruidAggregationQuery
+import com.yahoo.bard.webservice.table.ConstrainedTable
+
+import com.fasterxml.jackson.databind.ObjectMapper
+
+import spock.lang.Specification
+/**
+ * Test selector filter serialization.
+ */
+class SelectorFilterSpec extends Specification {
+
+    ObjectMapper objectMapper
+    LookupDimension dimension
+    DruidAggregationQuery druidQuery
+
+    ExtractionFunction extractionFunction
+
+    // Dimension missing due to lack of proper way to inject mock dimension value and therefore null is given
+    String expectedSerialization =
+            """
+                {
+                    "dimension": "foo",
+                    "type":"selector",
+                    "extractionFn":{
+                        "type":"registeredLookup",
+                        "lookup":"lookup",
+                        "retainMissingValue":false,
+                        "replaceMissingValueWith":"none",
+                        "injective":false,
+                        "optimize":false
+                    },
+                    "value":"value"
+                }
+            """
+
+    // Dimension missing due to lack of proper way to inject mock dimension value and therefore null is given
+    String expectedSerialization2 =
+            """
+                {
+                    "dimension": "foo",
+                    "type":"selector",
+                    "extractionFn":{
+                        "type":"registeredLookup",
+                        "lookup":"lookup2",
+                        "retainMissingValue":false,
+                        "replaceMissingValueWith":"none",
+                        "injective":false,
+                        "optimize":false
+                    },
+                    "value":"value"
+                }
+            """
+
+    def setup() {
+        objectMapper = new ObjectMapper()
+        dimension = Mock(LookupDimension)
+        extractionFunction = new RegisteredLookupExtractionFunction("lookup", false, "none", false, false)
+        TableDataSource dataSource = Mock(TableDataSource)
+        ConstrainedTable physicalTable = Mock(ConstrainedTable)
+        dataSource.getPhysicalTable() >> physicalTable
+        dataSource.getQuery() >> Optional.empty()
+        physicalTable.getPhysicalColumnName(_) >> "foo"
+        druidQuery = Mock(DruidAggregationQuery)
+        dimension.getExtractionFunction() >> Optional.of(extractionFunction)
+
+        druidQuery.getDataSource() >> dataSource
+
+    }
+
+    def "Serialization of dimension with lookups defaulted is correct"() {
+        given:
+        SelectorFilter filter = new SelectorFilter(dimension,"value")
+        druidQuery.getFilter() >> filter
+        String serializedFilter = objectMapper.writeValueAsString(druidQuery)
+
+        expect:
+        objectMapper.readTree(serializedFilter).get("filter") == objectMapper.readTree(expectedSerialization)
+    }
+
+    def "Serialization of dimension with lookups overriden is correct"() {
+        given:
+        SelectorFilter filter = new SelectorFilter(
+                dimension,
+                "value",
+                new RegisteredLookupExtractionFunction("lookup2", false, "none", false, false)
+        )
+        druidQuery.getFilter() >> filter
+        String serializedFilter = objectMapper.writeValueAsString(druidQuery)
+
+        expect:
+        objectMapper.readTree(serializedFilter).get("filter") == objectMapper.readTree(expectedSerialization2)
+    }
+}