Skip to content

Commit

Permalink
Add having filters (#553)
Browse files Browse the repository at this point in the history 
Support the dimSelector having filters
  • Loading branch information
@x4base @mistercrunch
x4base authored and mistercrunch committed Jun 21, 2016
1 parent 13095eb commit 485234b
Show file tree
Hide file tree
Showing 6 changed files with 195 additions and 92 deletions.
94 changes: 51 additions & 43 deletions caravel/assets/javascripts/explore.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,52 +23,53 @@ require('../node_modules/bootstrap-toggle/css/bootstrap-toggle.min.css');

var slice;

var getPanelClass = function (fieldPrefix) {
return (fieldPrefix === "flt" ? "filter" : "having") + "_panel";
};

function prepForm() {
var i = 1;
// Assigning the right id to form elements in filters
$("#filters > div").each(function () {
$(this).attr("id", function () {
return "flt_" + i;
});
$(this).find("#flt_col_0")
.attr("id", function () {
return "flt_col_" + i;
})
.attr("name", function () {
return "flt_col_" + i;
});
$(this).find("#flt_op_0")
.attr("id", function () {
return "flt_op_" + i;
})
.attr("name", function () {
return "flt_op_" + i;
});
$(this).find("#flt_eq_0")
.attr("id", function () {
return "flt_eq_" + i;
})
.attr("name", function () {
return "flt_eq_" + i;
});
i++;
var fixId = function ($filter, fieldPrefix, i) {
$filter.attr("id", function () {
return fieldPrefix + "_" + i;
});

["col", "op", "eq"].forEach(function (fieldMiddle) {
var fieldName = fieldPrefix + "_" + fieldMiddle;
$filter.find("#" + fieldName + "_0")
.attr("id", function () {
return fieldName + "_" + i;
})
.attr("name", function () {
return fieldName + "_" + i;
});
});
};

["flt", "having"].forEach(function (fieldPrefix) {
var i = 1;
$("#" + getPanelClass(fieldPrefix) + " #filters > div").each(function () {
fixId($(this), fieldPrefix, i);
i++;
});
});
}

function query(force, pushState) {
if (force === undefined) {
force = false;
}
if (pushState !== false) {
history.pushState({}, document.title, slice.querystring());
}
$('.query-and-save button').attr('disabled', 'disabled');
$('.btn-group.results span,a').attr('disabled', 'disabled');
if (force) { // Don't hide the alert message when the page is just loaded
$('div.alert').remove();
}
$('#is_cached').hide();
prepForm();
if (pushState !== false) {
// update the url after prepForm() fix the field ids
history.pushState({}, document.title, slice.querystring());
}
slice.render(force);
}

Expand Down Expand Up @@ -291,24 +292,26 @@ function initExploreView() {
$(".ui-helper-hidden-accessible").remove(); // jQuery-ui 1.11+ creates a div for every tooltip

function set_filters() {
for (var i = 1; i < 10; i++) {
var eq = px.getParam("flt_eq_" + i);
var col = px.getParam("flt_col_" + i);
if (eq !== '' && col !== '') {
add_filter(i);
["flt", "having"].forEach(function (prefix) {
for (var i = 1; i < 10; i++) {
var eq = px.getParam(prefix + "_eq_" + i);
var col = px.getParam(prefix + "_col_" + i);
if (eq !== '' && col !== '') {
add_filter(i, prefix);
}
}
}
});
}
set_filters();

function add_filter(i) {
var cp = $("#flt0").clone();
$(cp).appendTo("#filters");
function add_filter(i, fieldPrefix) {
var cp = $("#"+fieldPrefix+"0").clone();
$(cp).appendTo("#" + getPanelClass(fieldPrefix) + " #filters");
$(cp).show();
if (i !== undefined) {
$(cp).find("#flt_eq_0").val(px.getParam("flt_eq_" + i));
$(cp).find("#flt_op_0").val(px.getParam("flt_op_" + i));
$(cp).find("#flt_col_0").val(px.getParam("flt_col_" + i));
$(cp).find("#"+fieldPrefix+"_eq_0").val(px.getParam(fieldPrefix+"_eq_" + i));
$(cp).find("#"+fieldPrefix+"_op_0").val(px.getParam(fieldPrefix+"_op_" + i));
$(cp).find("#"+fieldPrefix+"_col_0").val(px.getParam(fieldPrefix+"_col_" + i));
}
$(cp).find('select').select2();
$(cp).find('.remove').click(function () {
Expand All @@ -324,7 +327,12 @@ function initExploreView() {
returnLocation.reload();
});

$("#plus").click(add_filter);
$("#filter_panel #plus").click(function () {
add_filter(undefined, "flt");
});
$("#having_panel #plus").click(function () {
add_filter(undefined, "having");
});
$("#btn_save").click(function () {
var slice_name = prompt("Name your slice!");
if (slice_name !== "" && slice_name !== null) {
Expand Down
38 changes: 25 additions & 13 deletions caravel/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,8 @@ def add_to_form(attrs):
setattr(QueryForm, attr, self.field_dict[attr])

filter_choices = self.choicify(['in', 'not in'])
having_op_choices = []
filter_prefixes = ['flt']
# datasource type specific form elements
datasource_classname = viz.datasource.__class__.__name__
time_fields = None
Expand Down Expand Up @@ -885,21 +887,31 @@ def add_to_form(attrs):
field_css_classes['granularity'] = ['form-control', 'select2_freeform']
field_css_classes['druid_time_origin'] = ['form-control', 'select2_freeform']
filter_choices = self.choicify(['in', 'not in', 'regex'])
having_op_choices = self.choicify(['>', '<', '=='])
filter_prefixes += ['having']
add_to_form(('since', 'until'))

filter_cols = viz.datasource.filterable_column_names or ['']
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
_('Filter 1'),
default=filter_cols[0],
choices=self.choicify(filter_cols)))
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
_('Filter 1'),
default='in',
choices=filter_choices))
setattr(
QueryForm, 'flt_eq_' + str(i),
TextField(_("Super"), default=''))
filter_cols = self.choicify(
viz.datasource.filterable_column_names or [''])
having_cols = filter_cols + viz.datasource.metrics_combo
for field_prefix in filter_prefixes:
is_having_filter = field_prefix == 'having'
col_choices = filter_cols if not is_having_filter else having_cols
op_choices = filter_choices if not is_having_filter else \
having_op_choices
for i in range(10):
setattr(QueryForm, field_prefix + '_col_' + str(i),
SelectField(
_('Filter 1'),
default=col_choices[0][0],
choices=col_choices))
setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField(
_('Filter 1'),
default=op_choices[0][0],
choices=op_choices))
setattr(
QueryForm, field_prefix + '_eq_' + str(i),
TextField(_("Super"), default=''))

if time_fields:
QueryForm.fieldsets = ({
Expand Down
92 changes: 63 additions & 29 deletions caravel/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from pydruid.client import PyDruid
from pydruid.utils.filters import Dimension, Filter
from pydruid.utils.postaggregator import Postaggregator
from pydruid.utils.having import Having, Aggregation
from six import string_types
from sqlalchemy import (
Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date,
Expand All @@ -41,7 +42,7 @@
import caravel
from caravel import app, db, get_session, utils, sm
from caravel.viz import viz_types
from caravel.utils import flasher, MetricPermException
from caravel.utils import flasher, MetricPermException, DimSelector

config = app.config

Expand Down Expand Up @@ -1191,38 +1192,15 @@ def recursive_get_fields(_conf):
post_aggregations=post_aggs,
intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
)
filters = None
for col, op, eq in filter:
cond = None
if op == '==':
cond = Dimension(col) == eq
elif op == '!=':
cond = ~(Dimension(col) == eq)
elif op in ('in', 'not in'):
fields = []
splitted = eq.split(',')
if len(splitted) > 1:
for s in eq.split(','):
s = s.strip()
fields.append(Dimension(col) == s)
cond = Filter(type="or", fields=fields)
else:
cond = Dimension(col) == eq
if op == 'not in':
cond = ~cond
elif op == 'regex':
cond = Filter(type="regex", pattern=eq, dimension=col)
if filters:
filters = Filter(type="and", fields=[
cond,
filters
])
else:
filters = cond

filters = self.get_filters(filter)
if filters:
qry['filter'] = filters

having_filters = self.get_having_filters(extras.get('having'))
if having_filters:
qry['having'] = having_filters

client = self.cluster.get_pydruid_client()
orig_filters = filters
if timeseries_limit and is_timeseries:
Expand Down Expand Up @@ -1303,6 +1281,62 @@ def recursive_get_fields(_conf):
query=query_str,
duration=datetime.now() - qry_start_dttm)

@staticmethod
def get_filters(raw_filters):
filters = None
for col, op, eq in raw_filters:
cond = None
if op == '==':
cond = Dimension(col) == eq
elif op == '!=':
cond = ~(Dimension(col) == eq)
elif op in ('in', 'not in'):
fields = []
splitted = eq.split(',')
if len(splitted) > 1:
for s in eq.split(','):
s = s.strip()
fields.append(Dimension(col) == s)
cond = Filter(type="or", fields=fields)
else:
cond = Dimension(col) == eq
if op == 'not in':
cond = ~cond
elif op == 'regex':
cond = Filter(type="regex", pattern=eq, dimension=col)
if filters:
filters = Filter(type="and", fields=[
cond,
filters
])
else:
filters = cond
return filters

def get_having_filters(self, raw_filters):
filters = None
for col, op, eq in raw_filters:
cond = None
if op == '==':
if col in self.column_names:
cond = DimSelector(dimension=col, value=eq)
else:
cond = Aggregation(col) == eq
elif op == '!=':
cond = ~(Aggregation(col) == eq)
elif op == '>':
cond = Aggregation(col) > eq
elif op == '<':
cond = Aggregation(col) < eq
if filters:
filters = Filter(type="and", fields=[
Having.build_having(cond),
Having.build_having(filters)
])
else:
filters = cond
return filters


class Log(Model):

Expand Down
37 changes: 36 additions & 1 deletion caravel/templates/caravel/explore.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@
</div>
</div>
{% endfor %}
<div class="panel panel-default">
<div id="filter_panel" class="panel panel-default">
<div class="panel-heading">
<span class="legend_label">{{ _("Filters") }}</span>
<i class="fa fa-info-circle" data-toggle="tooltip"
Expand All @@ -159,6 +159,41 @@
</button>
</div>
</div>


{% if form.having_col_0 %}
<div id="having_panel" class="panel panel-default">
<div class="panel-heading">
<span class="legend_label">Result Filters ("having" filters)</span>
<i class="fa fa-info-circle" data-toggle="tooltip"
data-placement="bottom"
title="The filters to apply after post-aggregation"></i>
<span class="collapser"> [-]</span>
</div>
<div class="panel-body">
<div id="having0" style="display: none;">
<span class="">{{ form.having_col_0(class_="form-control inc") }}</span>
<div class="row">
<span class="col col-sm-4">{{ form.having_op_0(class_="form-control inc") }}</span>
<span class="col col-sm-6">{{ form.having_eq_0(class_="form-control inc") }}</span>
<button type="button"
class="btn btn-default btn-sm remove"
aria-label="Delete filter">
<span class="fa fa-minus"
aria-hidden="true"></span>
</button>
</div>
</div>
<div id="filters"></div>
<button type="button" id="plus"
class="btn btn-default btn-sm"
aria-label="Add a filter">
<span class="fa fa-plus" aria-hidden="true"></span>
<span>Add filter</span>
</button>
</div>
</div>
{% endif %}
{{ form.slice_id() }}
{{ form.slice_name() }}
{{ form.collapsed_fieldsets() }}
Expand Down
13 changes: 13 additions & 0 deletions caravel/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from flask_appbuilder.security.sqla import models as ab_models
from markdown import markdown as md
from sqlalchemy.types import TypeDecorator, TEXT
from pydruid.utils.having import Having


class CaravelException(Exception):
Expand Down Expand Up @@ -77,6 +78,18 @@ def __get__(self, obj, objtype):
return functools.partial(self.__call__, obj)


class DimSelector(Having):
def __init__(self, **args):
# Just a hack to prevent any exceptions
Having.__init__(self, type='equalTo', aggregation=None, value=None)

self.having = {'having': {
'type': 'dimSelector',
'dimension': args['dimension'],
'value': args['value'],
}}


def list_minus(l, minus):
"""Returns l without what is in minus
Expand Down
Loading

0 comments on commit 485234b

Please sign in to comment.