Enable freeform-select with fetched column values for filter values (#…

…1697)

* Enable freeform-select with fetched column values for filter values
 - db migration to add filter_select_enabled
 - add freeform-multi option for Selectfield
 - modify formatFilter() function on query to accomodate filter-select

* Fix js tests

* Fix codeclimate issue

* Changes based on comments

* Add test for filter endpoint

* Extract out renderFilterFormField function from render

* Fix landscape issues

superset/assets/javascripts/explorev2/actions/exploreActions.js

@@ -93,6 +93,26 @@ export function changeFilter(filter, field, value) {
   return { type: CHANGE_FILTER, filter, field, value };
 }
 
+export function fetchFilterValues(datasource_type, datasource_id, filter, col) {
+  return function (dispatch) {
+    $.ajax({
+      type: 'GET',
+      url: `/superset/filter/${datasource_type}/${datasource_id}/${col}/`,
+      success: (data) => {
+        dispatch(changeFilter(
+          filter,
+          'choices',
+          Object.keys(data).map((k) => ([`'${data[k]}'`, `'${data[k]}'`]))
+          )
+        );
+      },
+      error() {
+        dispatch(changeFilter(filter, 'choices', []));
+      },
+    });
+  };
+}
+
 export const SET_FIELD_VALUE = 'SET_FIELD_VALUE';
 export function setFieldValue(datasource_type, key, value, label) {
   return { type: SET_FIELD_VALUE, datasource_type, key, value, label };

superset/assets/javascripts/explorev2/components/ControlPanelsContainer.jsx

@@ -102,6 +102,7 @@ class ControlPanelsContainer extends React.Component {
                 filters={this.props.form_data.filters}
                 actions={this.props.actions}
                 prefix={section.prefix}
+                datasource_id={this.props.form_data.datasource}
               />
             </ControlPanelSection>
           ))}

superset/assets/javascripts/explorev2/components/Filter.jsx

@@ -1,13 +1,16 @@
 import React from 'react';
-// import { Tab, Row, Col, Nav, NavItem } from 'react-bootstrap';
 import Select from 'react-select';
 import { Button } from 'react-bootstrap';
+import SelectField from './SelectField';
 
 const propTypes = {
   actions: React.PropTypes.object.isRequired,
   filterColumnOpts: React.PropTypes.array,
   prefix: React.PropTypes.string,
   filter: React.PropTypes.object.isRequired,
+  renderFilterSelect: React.PropTypes.bool,
+  datasource_type: React.PropTypes.string.isRequired,
+  datasource_id: React.PropTypes.number.isRequired,
 };
 
 const defaultProps = {
@@ -24,9 +27,22 @@ export default class Filter extends React.Component {
       opChoices,
     };
   }
+  componentWillMount() {
+    if (this.props.filter.col) {
+      this.props.actions.fetchFilterValues(
+        this.props.datasource_type,
+        this.props.datasource_id,
+        this.props.filter,
+        this.props.filter.col);
+    }
+  }
   changeCol(filter, colOpt) {
     const val = (colOpt) ? colOpt.value : null;
     this.props.actions.changeFilter(filter, 'col', val);
+    if (val) {
+      this.props.actions.fetchFilterValues(
+        this.props.datasource_type, this.props.datasource_id, filter, val);
+    }
   }
   changeOp(filter, opOpt) {
     const val = (opOpt) ? opOpt.value : null;
@@ -35,9 +51,35 @@ export default class Filter extends React.Component {
   changeValue(filter, event) {
     this.props.actions.changeFilter(filter, 'value', event.target.value);
   }
+  changeSelectValue(filter, name, value) {
+    this.props.actions.changeFilter(filter, 'value', value);
+  }
   removeFilter(filter) {
     this.props.actions.removeFilter(filter);
   }
+  renderFilterFormField() {
+    if (this.props.renderFilterSelect) {
+      return (
+        <SelectField
+          multi
+          freeForm
+          name="filter-value"
+          value={this.props.filter.value}
+          choices={this.props.filter.choices ? this.props.filter.choices : []}
+          onChange={this.changeSelectValue.bind(this, this.props.filter)}
+        />
+      );
+    }
+    return (
+      <input
+        type="text"
+        onChange={this.changeValue.bind(this, this.props.filter)}
+        value={this.props.filter.value}
+        className="form-control input-sm"
+        placeholder="Filter value"
+      />
+    );
+  }
   render() {
     return (
       <div>
@@ -65,13 +107,7 @@ export default class Filter extends React.Component {
             onChange={this.changeOp.bind(this, this.props.filter)}
           />
           <div className="col-lg-6">
-            <input
-              type="text"
-              onChange={this.changeValue.bind(this, this.props.filter)}
-              value={this.props.filter.value}
-              className="form-control input-sm"
-              placeholder="Filter value"
-            />
+            {this.renderFilterFormField()}
           </div>
           <div className="col-lg-2">
             <Button

superset/assets/javascripts/explorev2/components/Filters.jsx

@@ -7,9 +7,12 @@ import shortid from 'shortid';
 
 const propTypes = {
   actions: React.PropTypes.object.isRequired,
+  datasource_type: React.PropTypes.string.isRequired,
+  datasource_id: React.PropTypes.number.isRequired,
   filterColumnOpts: React.PropTypes.array,
   filters: React.PropTypes.array,
   prefix: React.PropTypes.string,
+  renderFilterSelect: React.PropTypes.bool,
 };
 
 const defaultProps = {
@@ -42,6 +45,9 @@ class Filters extends React.Component {
             actions={this.props.actions}
             prefix={this.props.prefix}
             filter={filter}
+            renderFilterSelect={this.props.renderFilterSelect}
+            datasource_type={this.props.datasource_type}
+            datasource_id={this.props.datasource_id}
           />
         );
       }
@@ -70,8 +76,10 @@ Filters.defaultProps = defaultProps;
 
 function mapStateToProps(state) {
   return {
+    datasource_type: state.datasource_type,
     filterColumnOpts: state.filterColumnOpts,
     filters: state.viz.form_data.filters,
+    renderFilterSelect: state.filter_select,
   };
 }
 

superset/assets/javascripts/explorev2/components/SelectField.jsx

@@ -58,8 +58,18 @@ export default class SelectField extends React.Component {
     if (this.props.freeForm) {
       // For FreeFormSelect, insert value into options if not exist
       const values = choices.map((c) => c[0]);
-      if (values.indexOf(this.props.value) === -1) {
-        options.push({ value: this.props.value, label: this.props.value });
+      if (this.props.value) {
+        if (typeof this.props.value === 'object') {
+          this.props.value.forEach((v) => {
+            if (values.indexOf(v) === -1) {
+              options.push({ value: v, label: v });
+            }
+          });
+        } else {
+          if (values.indexOf(this.props.value) === -1) {
+            options.push({ value: this.props.value, label: this.props.value });
+          }
+        }
       }
     }
 
@@ -77,13 +87,19 @@ export default class SelectField extends React.Component {
     //  Tab, comma or Enter will trigger a new option created for FreeFormSelect
     const selectWrap = this.props.freeForm ?
       (<Creatable {...selectProps} />) : (<Select {...selectProps} />);
-
+    if (this.props.label) {
+      return (
+        <div id={`formControlsSelect-${slugify(this.props.label)}`}>
+          <ControlLabelWithTooltip
+            label={this.props.label}
+            description={this.props.description}
+          />
+          {selectWrap}
+        </div>
+      );
+    }
     return (
-      <div id={`formControlsSelect-${slugify(this.props.label)}`}>
-        <ControlLabelWithTooltip
-          label={this.props.label}
-          description={this.props.description}
-        />
+      <div>
         {selectWrap}
       </div>
     );

superset/assets/javascripts/explorev2/index.jsx

@@ -25,6 +25,7 @@ const bootstrappedState = Object.assign(
   initialState(bootstrapData.viz.form_data.viz_type, bootstrapData.datasource_type), {
     can_edit: bootstrapData.can_edit,
     can_download: bootstrapData.can_download,
+    filter_select: bootstrapData.filter_select,
     datasources: bootstrapData.datasources,
     datasource_type: bootstrapData.datasource_type,
     viz: bootstrapData.viz,

superset/assets/javascripts/explorev2/stores/store.js

@@ -43,6 +43,7 @@ export function initialState(vizType = 'table', datasourceType = 'table') {
     datasources: null,
     datasource_type: null,
     filterColumnOpts: [],
+    filter_select: false,
     fields,
     viz: defaultViz(vizType, datasourceType),
     isStarred: false,

superset/assets/spec/javascripts/explorev2/components/Filter_spec.js

@@ -8,7 +8,9 @@ import { shallow } from 'enzyme';
 import Filter from '../../../../javascripts/explorev2/components/Filter';
 
 const defaultProps = {
-  actions: {},
+  actions: {
+    fetchFilterValues: () => ({}),
+  },
   filterColumnOpts: ['country_name'],
   filter: {
     id: 1,

superset/migrations/versions/f1f2d4af5b90_.py

@@ -0,0 +1,25 @@
+"""Enable Filter Select
+
+Revision ID: f1f2d4af5b90
+Revises: e46f2d27a08e
+Create Date: 2016-11-23 10:27:18.517919
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f1f2d4af5b90'
+down_revision = 'e46f2d27a08e'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('datasources', sa.Column('filter_select_enabled',
+                                           sa.Boolean(), default=False))
+    op.add_column('tables', sa.Column('filter_select_enabled',
+                                      sa.Boolean(), default=False))
+
+def downgrade():
+    op.drop_column('tables', 'filter_select_enabled')
+    op.drop_column('datasources', 'filter_select_enabled')

superset/models.py

@@ -32,6 +32,7 @@
 from flask_babel import lazy_gettext as _
 
 from pydruid.client import PyDruid
+from pydruid.utils.aggregators import count
 from pydruid.utils.filters import Dimension, Filter
 from pydruid.utils.postaggregator import Postaggregator
 from pydruid.utils.having import Aggregation
@@ -866,6 +867,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable, ImportMixin):
     default_endpoint = Column(Text)
     database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
     is_featured = Column(Boolean, default=False)
+    filter_select_enabled = Column(Boolean, default=False)
     user_id = Column(Integer, ForeignKey('ab_user.id'))
     owner = relationship('User', backref='tables', foreign_keys=[user_id])
     database = relationship(
@@ -977,6 +979,45 @@ def get_col(self, col_name):
             if col_name == col.column_name:
                 return col
 
+    def values_for_column(self,
+                          column_name,
+                          from_dttm,
+                          to_dttm,
+                          limit=500):
+        """Runs query against sqla to retrieve some
+        sample values for the given column.
+        """
+        granularity = self.main_dttm_col
+
+        cols = {col.column_name: col for col in self.columns}
+        target_col = cols[column_name]
+
+        tbl = table(self.table_name)
+        qry = select([target_col.sqla_col])
+        qry = qry.select_from(tbl)
+        qry = qry.distinct(column_name)
+        qry = qry.limit(limit)
+
+        if granularity:
+            dttm_col = cols[granularity]
+            timestamp = dttm_col.sqla_col.label('timestamp')
+            time_filter = [
+                timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)),
+                timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)),
+            ]
+            qry = qry.where(and_(*time_filter))
+
+        engine = self.database.get_sqla_engine()
+        sql = "{}".format(
+            qry.compile(
+                engine, compile_kwargs={"literal_binds": True}, ),
+        )
+
+        return pd.read_sql_query(
+            sql=sql,
+            con=engine
+        )
+
     def query(  # sqla
             self, groupby, metrics,
             granularity,
@@ -1594,6 +1635,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
     datasource_name = Column(String(255), unique=True)
     is_featured = Column(Boolean, default=False)
     is_hidden = Column(Boolean, default=False)
+    filter_select_enabled = Column(Boolean, default=False)
     description = Column(Text)
     default_endpoint = Column(Text)
     user_id = Column(Integer, ForeignKey('ab_user.id'))
@@ -1930,6 +1972,35 @@ def granularity(period_name, timezone=None, origin=None):
                 period_name).total_seconds() * 1000
         return granularity
 
+    def values_for_column(self,
+                          column_name,
+                          from_dttm,
+                          to_dttm,
+                          limit=500):
+        """Retrieve some values for the given column"""
+        # TODO: Use Lexicographic TopNMeticSpec onces supported by PyDruid
+        from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+        to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+
+        qry = dict(
+            datasource=self.datasource_name,
+            granularity="all",
+            intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
+            aggregations=dict(count=count("count")),
+            dimension=column_name,
+            metric="count",
+            threshold=limit,
+        )
+
+        client = self.cluster.get_pydruid_client()
+        client.topn(**qry)
+        df = client.export_pandas()
+
+        if df is None or df.size == 0:
+            raise Exception(_("No data was returned."))
+
+        return df
+
     def query(  # druid
             self, groupby, metrics,
             granularity,