diff --git a/.pylintrc b/.pylintrc index 27b41d7b42e0c..8a0e0ed78f612 100644 --- a/.pylintrc +++ b/.pylintrc @@ -102,7 +102,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme good-names=i,j,k,ex,Run,_,d,e,v,o,l,x,ts # Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +bad-names=foo,bar,baz,toto,tutu,tata,d,fd # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. diff --git a/superset/assets/images/viz_thumbnails/deck_grid.png b/superset/assets/images/viz_thumbnails/deck_grid.png new file mode 100644 index 0000000000000..cd9396510633d Binary files /dev/null and b/superset/assets/images/viz_thumbnails/deck_grid.png differ diff --git a/superset/assets/images/viz_thumbnails/deck_hex.png b/superset/assets/images/viz_thumbnails/deck_hex.png new file mode 100644 index 0000000000000..31feff5c8fb08 Binary files /dev/null and b/superset/assets/images/viz_thumbnails/deck_hex.png differ diff --git a/superset/assets/images/viz_thumbnails/deck_scatter.png b/superset/assets/images/viz_thumbnails/deck_scatter.png new file mode 100644 index 0000000000000..11f38ccc8dbf3 Binary files /dev/null and b/superset/assets/images/viz_thumbnails/deck_scatter.png differ diff --git a/superset/assets/images/viz_thumbnails/deck_screengrid.png b/superset/assets/images/viz_thumbnails/deck_screengrid.png new file mode 100644 index 0000000000000..d5da29c99be9f Binary files /dev/null and b/superset/assets/images/viz_thumbnails/deck_screengrid.png differ diff --git a/superset/assets/javascripts/explore/components/controls/FixedOrMetricControl.jsx b/superset/assets/javascripts/explore/components/controls/FixedOrMetricControl.jsx new file mode 100644 index 0000000000000..2c93fd18482ff --- /dev/null +++ b/superset/assets/javascripts/explore/components/controls/FixedOrMetricControl.jsx @@ -0,0 +1,118 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Label, Popover, OverlayTrigger } from 'react-bootstrap'; + +import controls from '../../stores/controls'; +import TextControl from './TextControl'; +import SelectControl from './SelectControl'; +import ControlHeader from '../ControlHeader'; +import PopoverSection from '../../../components/PopoverSection'; + +const propTypes = { + onChange: PropTypes.func, + value: PropTypes.object, + isFloat: PropTypes.bool, + datasource: PropTypes.object, + default: PropTypes.object, +}; + +const defaultProps = { + onChange: () => {}, + default: { type: 'fix', value: 5 }, +}; + +export default class FixedOrMetricControl extends React.Component { + constructor(props) { + super(props); + this.onChange = this.onChange.bind(this); + const type = props.value ? props.value.type : props.default.type || 'fix'; + const value = props.value ? props.value.value : props.default.value || '100'; + this.state = { + type, + fixedValue: type === 'fix' ? value : '', + metricValue: type === 'metric' ? value : null, + }; + } + onChange() { + this.props.onChange({ + type: this.state.type, + value: this.state.type === 'fix' ? this.state.fixedValue : this.state.metricValue, + }); + } + setType(type) { + this.setState({ type }, this.onChange); + } + setFixedValue(fixedValue) { + this.setState({ fixedValue }, this.onChange); + } + setMetric(metricValue) { + this.setState({ metricValue }, this.onChange); + } + renderPopover() { + const value = this.props.value || this.props.default; + const type = value.type || 'fix'; + const metrics = this.props.datasource ? this.props.datasource.metrics : null; + return ( + +
+ + + + + + +
+ + ); + } + render() { + return ( +
+ + + + +
+ ); + } +} + +FixedOrMetricControl.propTypes = propTypes; +FixedOrMetricControl.defaultProps = defaultProps; diff --git a/superset/assets/javascripts/explore/components/controls/SelectControl.jsx b/superset/assets/javascripts/explore/components/controls/SelectControl.jsx index a82995d70e4fa..6441b71c8de1b 100644 --- a/superset/assets/javascripts/explore/components/controls/SelectControl.jsx +++ b/superset/assets/javascripts/explore/components/controls/SelectControl.jsx @@ -17,6 +17,7 @@ const propTypes = { multi: PropTypes.bool, name: PropTypes.string.isRequired, onChange: PropTypes.func, + onFocus: PropTypes.func, value: PropTypes.oneOfType([PropTypes.string, PropTypes.number, PropTypes.array]), showHeader: PropTypes.bool, optionRenderer: PropTypes.func, @@ -34,6 +35,7 @@ const defaultProps = { label: null, multi: false, onChange: () => {}, + onFocus: () => {}, showHeader: true, optionRenderer: opt => opt.label, valueRenderer: opt => opt.label, @@ -115,6 +117,7 @@ export default class SelectControl extends React.PureComponent { clearable: this.props.clearable, isLoading: this.props.isLoading, onChange: this.onChange, + onFocus: this.props.onFocus, optionRenderer: VirtualizedRendererWrap(this.props.optionRenderer), valueRenderer: this.props.valueRenderer, selectComponent: this.props.freeForm ? Creatable : Select, diff --git a/superset/assets/javascripts/explore/components/controls/TextControl.jsx b/superset/assets/javascripts/explore/components/controls/TextControl.jsx index 4fe558e0524ac..bfe3f99177cab 100644 --- a/superset/assets/javascripts/explore/components/controls/TextControl.jsx +++ b/superset/assets/javascripts/explore/components/controls/TextControl.jsx @@ -5,10 +5,8 @@ import * as v from '../../validators'; import ControlHeader from '../ControlHeader'; const propTypes = { - name: PropTypes.string.isRequired, - label: PropTypes.string, - description: PropTypes.string, onChange: PropTypes.func, + onFocus: PropTypes.func, value: PropTypes.oneOfType([ PropTypes.string, PropTypes.number, @@ -18,9 +16,8 @@ const propTypes = { }; const defaultProps = { - label: null, - description: null, onChange: () => {}, + onFocus: () => {}, value: '', isInt: false, isFloat: false, @@ -64,6 +61,7 @@ export default class TextControl extends React.Component { type="text" placeholder="" onChange={this.onChange} + onFocus={this.props.onFocus} value={value} /> diff --git a/superset/assets/javascripts/explore/components/controls/ViewportControl.jsx b/superset/assets/javascripts/explore/components/controls/ViewportControl.jsx new file mode 100644 index 0000000000000..ba134e256f6a0 --- /dev/null +++ b/superset/assets/javascripts/explore/components/controls/ViewportControl.jsx @@ -0,0 +1,97 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Label, Popover, OverlayTrigger } from 'react-bootstrap'; +import { decimal2sexagesimal } from 'geolib'; + +import TextControl from './TextControl'; +import ControlHeader from '../ControlHeader'; + +const PARAMS = [ + 'longitude', + 'latitude', + 'zoom', + 'bearing', + 'pitch', +]; + +const propTypes = { + onChange: PropTypes.func.isRequired, + value: PropTypes.object, + default: PropTypes.object, +}; + +const defaultProps = { + onChange: () => {}, + default: { type: 'fix', value: 5 }, + value: { + longitude: 6.85236157047845, + latitude: 31.222656842808707, + zoom: 1, + bearing: 0, + pitch: 0, + }, +}; + +export default class ViewportControl extends React.Component { + constructor(props) { + super(props); + this.onChange = this.onChange.bind(this); + } + onChange(ctrl, value) { + this.props.onChange({ + ...this.props.value, + [ctrl]: value, + }); + } + renderTextControl(ctrl) { + return ( +
+ {ctrl} + +
+ ); + } + renderPopover() { + return ( + + {PARAMS.map(ctrl => this.renderTextControl(ctrl))} + + ); + } + renderLabel() { + if (this.props.value.longitude && this.props.value.latitude) { + return ( + decimal2sexagesimal(this.props.value.longitude) + + ' | ' + + decimal2sexagesimal(this.props.value.latitude) + ); + } + return 'N/A'; + } + render() { + return ( +
+ + + + +
+ ); + } +} + +ViewportControl.propTypes = propTypes; +ViewportControl.defaultProps = defaultProps; diff --git a/superset/assets/javascripts/explore/components/controls/index.js b/superset/assets/javascripts/explore/components/controls/index.js index 876bc4a1c631c..094a26b32318d 100644 --- a/superset/assets/javascripts/explore/components/controls/index.js +++ b/superset/assets/javascripts/explore/components/controls/index.js @@ -6,12 +6,14 @@ import ColorSchemeControl from './ColorSchemeControl'; import DatasourceControl from './DatasourceControl'; import DateFilterControl from './DateFilterControl'; import FilterControl from './FilterControl'; +import FixedOrMetricControl from './FixedOrMetricControl'; import HiddenControl from './HiddenControl'; import SelectAsyncControl from './SelectAsyncControl'; import SelectControl from './SelectControl'; import TextAreaControl from './TextAreaControl'; import TextControl from './TextControl'; import TimeSeriesColumnControl from './TimeSeriesColumnControl'; +import ViewportControl from './ViewportControl'; import VizTypeControl from './VizTypeControl'; const controlMap = { @@ -23,12 +25,14 @@ const controlMap = { DatasourceControl, DateFilterControl, FilterControl, + FixedOrMetricControl, HiddenControl, SelectAsyncControl, SelectControl, TextAreaControl, TextControl, TimeSeriesColumnControl, + ViewportControl, VizTypeControl, }; export default controlMap; diff --git a/superset/assets/javascripts/explore/stores/controls.jsx b/superset/assets/javascripts/explore/stores/controls.jsx index e3da06b621bd7..cd9733e97dfa1 100644 --- a/superset/assets/javascripts/explore/stores/controls.jsx +++ b/superset/assets/javascripts/explore/stores/controls.jsx @@ -1,7 +1,7 @@ import React from 'react'; import { formatSelectOptionsForRange, formatSelectOptions } from '../../modules/utils'; import * as v from '../validators'; -import { ALL_COLOR_SCHEMES, spectrums } from '../../modules/colors'; +import { colorPrimary, ALL_COLOR_SCHEMES, spectrums } from '../../modules/colors'; import MetricOption from '../../components/MetricOption'; import ColumnOption from '../../components/ColumnOption'; import OptionDescription from '../../components/OptionDescription'; @@ -135,6 +135,14 @@ export const controls = { }), }, + color_picker: { + label: t('Fixed Color'), + description: t('Use this to define a static color for all circles'), + type: 'ColorPickerControl', + default: colorPrimary, + renderTrigger: true, + }, + annotation_layers: { type: 'SelectAsyncControl', multi: true, @@ -424,6 +432,12 @@ export const controls = { }, groupby: groupByControl, + dimension: Object.assign({}, groupByControl, { + label: t('Dimension'), + description: t('Select a dimension'), + multi: false, + default: null, + }), columns: Object.assign({}, groupByControl, { label: t('Columns'), @@ -730,6 +744,14 @@ export const controls = { 'with the [Periods] text box'), }, + multiplier: { + type: 'TextControl', + label: t('Multiplier'), + isFloat: true, + default: 1, + description: t('Factor to multiply the metric by'), + }, + rolling_periods: { type: 'TextControl', label: t('Periods'), @@ -738,6 +760,15 @@ export const controls = { 'relative to the time granularity selected'), }, + grid_size: { + type: 'TextControl', + label: t('Grid Size'), + renderTrigger: true, + default: 20, + isInt: true, + description: t('Defines the grid size in pixels'), + }, + min_periods: { type: 'TextControl', label: t('Min Periods'), @@ -1043,6 +1074,14 @@ export const controls = { ), }, + extruded: { + type: 'CheckboxControl', + label: t('Extruded'), + renderTrigger: true, + default: true, + description: ('Whether to make the grid 3D'), + }, + show_brush: { type: 'CheckboxControl', label: t('Range Filter'), @@ -1255,6 +1294,7 @@ export const controls = { mapbox_style: { type: 'SelectControl', label: t('Map Style'), + renderTrigger: true, choices: [ ['mapbox://styles/mapbox/streets-v9', 'Streets'], ['mapbox://styles/mapbox/dark-v9', 'Dark'], @@ -1288,6 +1328,15 @@ export const controls = { 'number of points (>1000) will cause lag.'), }, + point_radius_fixed: { + type: 'FixedOrMetricControl', + label: t('Point Size'), + description: t('Fixed point radius'), + mapStateToProps: state => ({ + datasource: state.datasource, + }), + }, + point_radius: { type: 'SelectControl', label: t('Point Radius'), @@ -1308,6 +1357,22 @@ export const controls = { description: t('The unit of measure for the specified point radius'), }, + point_unit: { + type: 'SelectControl', + label: t('Point Unit'), + default: 'square_m', + clearable: false, + choices: [ + ['square_m', 'Square meters'], + ['square_km', 'Square kilometers'], + ['square_miles', 'Square miles'], + ['radius_m', 'Radius in meters'], + ['radius_km', 'Radius in kilometers'], + ['radius_miles', 'Radius in miles'], + ], + description: t('The unit of measure for the specified point radius'), + }, + global_opacity: { type: 'TextControl', label: t('Opacity'), @@ -1317,6 +1382,20 @@ export const controls = { 'Between 0 and 1.'), }, + viewport: { + type: 'ViewportControl', + label: t('Viewport'), + renderTrigger: true, + description: t('Parameters related to the view and perspective on the map'), + default: { + longitude: 6.85236157047845, + latitude: 31.222656842808707, + zoom: 1, + bearing: 0, + pitch: 0, + }, + }, + viewport_zoom: { type: 'TextControl', label: t('Zoom'), diff --git a/superset/assets/javascripts/explore/stores/visTypes.js b/superset/assets/javascripts/explore/stores/visTypes.js index 94115bb07622c..ea1b9ffd7f9af 100644 --- a/superset/assets/javascripts/explore/stores/visTypes.js +++ b/superset/assets/javascripts/explore/stores/visTypes.js @@ -294,6 +294,153 @@ export const visTypes = { }, }, + deck_hex: { + label: t('Deck.gl - Hexagons'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['all_columns_x', 'all_columns_y'], + ['groupby', 'size'], + ['row_limit'], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + ['color_picker', null], + ['grid_size', 'extruded'], + ], + }, + ], + controlOverrides: { + size: { + label: t('Height'), + description: t('Metric used to control height'), + validators: [v.nonEmpty], + }, + }, + }, + + deck_grid: { + label: t('Deck.gl - Grid'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['all_columns_x', 'all_columns_y'], + ['groupby', 'size'], + ['row_limit'], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + ['color_picker', null], + ['grid_size', 'extruded'], + ], + }, + ], + controlOverrides: { + size: { + label: t('Height'), + description: t('Metric used to control height'), + validators: [v.nonEmpty], + }, + }, + }, + + deck_screengrid: { + label: t('Deck.gl - Screen grid'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['all_columns_x', 'all_columns_y'], + ['groupby', 'size'], + ['row_limit'], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + ], + }, + { + label: t('Grid'), + controlSetRows: [ + ['grid_size', 'color_picker'], + ], + }, + ], + controlOverrides: { + size: { + label: t('Weight'), + description: t("Metric used as a weight for the grid's coloring"), + validators: [v.nonEmpty], + }, + }, + }, + + deck_scatter: { + label: t('Deck.gl - Scatter plot'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['all_columns_x', 'all_columns_y'], + ['groupby'], + ['row_limit'], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + ], + }, + { + label: t('Point Size'), + controlSetRows: [ + ['point_radius_fixed', 'point_unit'], + ['multiplier', null], + ], + }, + { + label: t('Point Color'), + controlSetRows: [ + ['color_picker', null], + ['dimension', 'color_scheme'], + ], + }, + ], + controlOverrides: { + all_columns_x: { + label: t('Longitude Column'), + validators: [v.nonEmpty], + }, + all_columns_y: { + label: t('Latitude Column'), + validators: [v.nonEmpty], + }, + dimension: { + label: t('Categorical Color'), + description: t('Pick a dimension from which categorical colors are defined'), + }, + }, + }, + area: { label: t('Time Series - Stacked'), requiresTime: true, @@ -1062,9 +1209,9 @@ export const visTypes = { { label: t('Viewport'), controlSetRows: [ - ['viewport_longitude'], - ['viewport_latitude'], - ['viewport_zoom'], + ['viewport_longitude', 'viewport_latitude'], + ['viewport_zoom', 'viewport_bearing'], + ['viewport_pitch', null], ], }, ], diff --git a/superset/assets/javascripts/modules/colors.js b/superset/assets/javascripts/modules/colors.js index 663fd6e4947c3..0c3d06a09d549 100644 --- a/superset/assets/javascripts/modules/colors.js +++ b/superset/assets/javascripts/modules/colors.js @@ -142,3 +142,13 @@ export const colorScalerFactory = function (colors, data, accessor, extents) { const points = colors.map((col, i) => ext[0] + (i * chunkSize)); return d3.scale.linear().domain(points).range(colors).clamp(true); }; + +export function hexToRGB(hex, alpha = 255) { + if (!hex) { + return [0, 0, 0, alpha]; + } + const r = parseInt(hex.slice(1, 3), 16); + const g = parseInt(hex.slice(3, 5), 16); + const b = parseInt(hex.slice(5, 7), 16); + return [r, g, b, alpha]; +} diff --git a/superset/assets/javascripts/modules/geo.js b/superset/assets/javascripts/modules/geo.js new file mode 100644 index 0000000000000..7cac25251bb7c --- /dev/null +++ b/superset/assets/javascripts/modules/geo.js @@ -0,0 +1,18 @@ + +const METER_TO_MILE = 1609.34; +export function unitToRadius(unit, num) { + if (unit === 'square_m') { + return Math.sqrt(num / Math.PI); + } else if (unit === 'radius_m') { + return num; + } else if (unit === 'radius_km') { + return num * 1000; + } else if (unit === 'radius_miles') { + return num * METER_TO_MILE; + } else if (unit === 'square_km') { + return Math.sqrt(num / Math.PI) * 1000; + } else if (unit === 'square_miles') { + return Math.sqrt(num / Math.PI) * METER_TO_MILE; + } + return null; +} diff --git a/superset/assets/package.json b/superset/assets/package.json index 6f6068d6e5247..21274cfdbb86e 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -55,11 +55,14 @@ "d3-tip": "^0.6.7", "datamaps": "^0.5.8", "datatables.net-bs": "^1.10.15", + "deck.gl": "^4.1.5", "distributions": "^1.0.0", + "geolib": "^2.0.24", "immutable": "^3.8.2", "jed": "^1.1.1", "jquery": "3.1.1", "lodash.throttle": "^4.1.1", + "luma.gl": "^4.0.5", "moment": "2.18.1", "mustache": "^2.2.1", "nvd3": "1.8.6", diff --git a/superset/assets/spec/javascripts/explore/components/FixedOrMetricControl_spec.jsx b/superset/assets/spec/javascripts/explore/components/FixedOrMetricControl_spec.jsx new file mode 100644 index 0000000000000..6f5a1aa5b5c60 --- /dev/null +++ b/superset/assets/spec/javascripts/explore/components/FixedOrMetricControl_spec.jsx @@ -0,0 +1,39 @@ +/* eslint-disable no-unused-expressions */ +import React from 'react'; +import { expect } from 'chai'; +import { describe, it, beforeEach } from 'mocha'; +import { shallow } from 'enzyme'; +import { OverlayTrigger } from 'react-bootstrap'; + +import FixedOrMetricControl from + '../../../../javascripts/explore/components/controls/FixedOrMetricControl'; +import SelectControl from + '../../../../javascripts/explore/components/controls/SelectControl'; +import TextControl from + '../../../../javascripts/explore/components/controls/TextControl'; +import ControlHeader from '../../../../javascripts/explore/components/ControlHeader'; + +const defaultProps = { + value: { }, +}; + +describe('FixedOrMetricControl', () => { + let wrapper; + let inst; + beforeEach(() => { + wrapper = shallow(); + inst = wrapper.instance(); + }); + + it('renders a OverlayTrigger', () => { + const controlHeader = wrapper.find(ControlHeader); + expect(controlHeader).to.have.lengthOf(1); + expect(wrapper.find(OverlayTrigger)).to.have.length(1); + }); + + it('renders a TextControl and a SelectControl', () => { + const popOver = shallow(inst.renderPopover()); + expect(popOver.find(TextControl)).to.have.lengthOf(1); + expect(popOver.find(SelectControl)).to.have.lengthOf(1); + }); +}); diff --git a/superset/assets/spec/javascripts/explore/components/ViewportControl_spec.jsx b/superset/assets/spec/javascripts/explore/components/ViewportControl_spec.jsx new file mode 100644 index 0000000000000..9864d83890ca6 --- /dev/null +++ b/superset/assets/spec/javascripts/explore/components/ViewportControl_spec.jsx @@ -0,0 +1,46 @@ +/* eslint-disable no-unused-expressions */ +import React from 'react'; +import { expect } from 'chai'; +import { describe, it, beforeEach } from 'mocha'; +import { shallow } from 'enzyme'; +import { OverlayTrigger, Label } from 'react-bootstrap'; + +import ViewportControl from + '../../../../javascripts/explore/components/controls/ViewportControl'; +import TextControl from + '../../../../javascripts/explore/components/controls/TextControl'; +import ControlHeader from '../../../../javascripts/explore/components/ControlHeader'; + +const defaultProps = { + value: { + longitude: 6.85236157047845, + latitude: 31.222656842808707, + zoom: 1, + bearing: 0, + pitch: 0, + }, +}; + +describe('ViewportControl', () => { + let wrapper; + let inst; + beforeEach(() => { + wrapper = shallow(); + inst = wrapper.instance(); + }); + + it('renders a OverlayTrigger', () => { + const controlHeader = wrapper.find(ControlHeader); + expect(controlHeader).to.have.lengthOf(1); + expect(wrapper.find(OverlayTrigger)).to.have.length(1); + }); + + it('renders a Popover with 5 TextControl', () => { + const popOver = shallow(inst.renderPopover()); + expect(popOver.find(TextControl)).to.have.lengthOf(5); + }); + + it('renders a summary in the label', () => { + expect(wrapper.find(Label).first().render().text()).to.equal('6° 51\' 8.50" | 31° 13\' 21.56"'); + }); +}); diff --git a/superset/assets/spec/javascripts/modules/colors_spec.jsx b/superset/assets/spec/javascripts/modules/colors_spec.jsx index 558547101a06e..31ccea8326670 100644 --- a/superset/assets/spec/javascripts/modules/colors_spec.jsx +++ b/superset/assets/spec/javascripts/modules/colors_spec.jsx @@ -1,7 +1,7 @@ import { it, describe } from 'mocha'; import { expect } from 'chai'; -import { ALL_COLOR_SCHEMES, getColorFromScheme } from '../../../javascripts/modules/colors'; +import { ALL_COLOR_SCHEMES, getColorFromScheme, hexToRGB } from '../../../javascripts/modules/colors'; describe('colors', () => { it('default to bnbColors', () => { @@ -19,4 +19,13 @@ describe('colors', () => { expect(color1).to.equal(color3); expect(color4).to.equal(ALL_COLOR_SCHEMES.bnbColors[1]); }); + + it('hexToRGB converts properly', () => { + expect(hexToRGB('#FFFFFF')).to.have.same.members([255, 255, 255, 255]); + expect(hexToRGB('#000000')).to.have.same.members([0, 0, 0, 255]); + expect(hexToRGB('#FF0000')).to.have.same.members([255, 0, 0, 255]); + expect(hexToRGB('#00FF00')).to.have.same.members([0, 255, 0, 255]); + expect(hexToRGB('#0000FF')).to.have.same.members([0, 0, 255, 255]); + expect(hexToRGB('#FF0000', 128)).to.have.same.members([255, 0, 0, 128]); + }); }); diff --git a/superset/assets/spec/javascripts/modules/geo_spec.jsx b/superset/assets/spec/javascripts/modules/geo_spec.jsx new file mode 100644 index 0000000000000..758bf15a135d2 --- /dev/null +++ b/superset/assets/spec/javascripts/modules/geo_spec.jsx @@ -0,0 +1,27 @@ +import { it, describe } from 'mocha'; +import { expect } from 'chai'; + +import { unitToRadius } from '../../../javascripts/modules/geo'; + +const METER_TO_MILE = 1609.34; + +describe('unitToRadius', () => { + it('converts to square meters', () => { + expect(unitToRadius('square_m', 4 * Math.PI)).to.equal(2); + }); + it('converts to square meters', () => { + expect(unitToRadius('square_km', 25 * Math.PI)).to.equal(5000); + }); + it('converts to radius meters', () => { + expect(unitToRadius('radius_m', 1000)).to.equal(1000); + }); + it('converts to radius km', () => { + expect(unitToRadius('radius_km', 1)).to.equal(1000); + }); + it('converts to radius miles', () => { + expect(unitToRadius('radius_miles', 1)).to.equal(METER_TO_MILE); + }); + it('converts to square miles', () => { + expect(unitToRadius('square_miles', 25 * Math.PI)).to.equal(5000 * (METER_TO_MILE / 1000)); + }); +}); diff --git a/superset/assets/visualizations/deckgl/DeckGLContainer.jsx b/superset/assets/visualizations/deckgl/DeckGLContainer.jsx new file mode 100644 index 0000000000000..a4f9d4e36de09 --- /dev/null +++ b/superset/assets/visualizations/deckgl/DeckGLContainer.jsx @@ -0,0 +1,91 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import MapGL from 'react-map-gl'; +import DeckGL from 'deck.gl'; + +const propTypes = { + viewport: PropTypes.object.isRequired, + layers: PropTypes.array.isRequired, + setControlValue: PropTypes.func.isRequired, + mapStyle: PropTypes.string, + mapboxApiAccessToken: PropTypes.string.isRequired, + onViewportChange: PropTypes.func, +}; +const defaultProps = { + mapStyle: 'light', + onViewportChange: () => {}, +}; + +export default class DeckGLContainer extends React.Component { + constructor(props) { + super(props); + this.state = { + viewport: props.viewport, + }; + this.tick = this.tick.bind(this); + } + componentWillMount() { + const timer = setInterval(this.tick, 1000); + this.setState({ timer }); + } + componentWillReceiveProps(nextProps) { + this.setState({ + viewport: { + ...nextProps.viewport, + }, + }); + } + componentWillUnmount() { + this.clearInterval(this.state.timer); + } + onViewportChange(viewport) { + const vp = Object.assign({}, viewport); + delete vp.width; + delete vp.height; + const newVp = { ...this.state.viewport, ...vp }; + + this.setState({ + viewport: newVp, + }); + this.props.onViewportChange(newVp); + } + tick() { + // Limiting updating viewport controls through Redux at most 1*sec + if (this.state.previousViewport !== this.state.viewport) { + const setCV = this.props.setControlValue; + const vp = this.state.viewport; + if (setCV) { + setCV('viewport', vp); + } + + this.setState({ previousViewport: this.state.viewport }); + } + } + layers() { + // Support for layer factory + if (this.props.layers.some(l => typeof l === 'function')) { + return this.props.layers.map(l => typeof l === 'function' ? l() : l); + } + return this.props.layers; + } + render() { + const { viewport } = this.state; + return ( + + + + ); + } +} + +DeckGLContainer.propTypes = propTypes; +DeckGLContainer.defaultProps = defaultProps; diff --git a/superset/assets/visualizations/deckgl/grid.jsx b/superset/assets/visualizations/deckgl/grid.jsx new file mode 100644 index 0000000000000..496728e52d385 --- /dev/null +++ b/superset/assets/visualizations/deckgl/grid.jsx @@ -0,0 +1,43 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { GridLayer } from 'deck.gl'; + +import DeckGLContainer from './DeckGLContainer'; + +function deckScreenGridLayer(slice, payload, setControlValue) { + const fd = slice.formData; + const c = fd.color_picker; + const data = payload.data.features.map(d => ({ + ...d, + color: [c.r, c.g, c.b, 255 * c.a], + })); + + const layer = new GridLayer({ + id: 'screengrid-layer', + data, + pickable: true, + cellSize: fd.grid_size, + minColor: [0, 0, 0, 0], + extruded: fd.extruded, + maxColor: [c.r, c.g, c.b, 255 * c.a], + outline: false, + getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + }); + const viewport = { + ...fd.viewport, + width: slice.width(), + height: slice.height(), + }; + ReactDOM.render( + , + document.getElementById(slice.containerId), + ); +} +module.exports = deckScreenGridLayer; diff --git a/superset/assets/visualizations/deckgl/hex.jsx b/superset/assets/visualizations/deckgl/hex.jsx new file mode 100644 index 0000000000000..22ae9ac11b645 --- /dev/null +++ b/superset/assets/visualizations/deckgl/hex.jsx @@ -0,0 +1,43 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { HexagonLayer } from 'deck.gl'; + +import DeckGLContainer from './DeckGLContainer'; + +function deckHex(slice, payload, setControlValue) { + const fd = slice.formData; + const c = fd.color_picker; + const data = payload.data.features.map(d => ({ + ...d, + color: [c.r, c.g, c.b, 255 * c.a], + })); + + const layer = new HexagonLayer({ + id: 'screengrid-layer', + data, + pickable: true, + radius: fd.grid_size, + minColor: [0, 0, 0, 0], + extruded: fd.extruded, + maxColor: [c.r, c.g, c.b, 255 * c.a], + outline: false, + getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + }); + const viewport = { + ...fd.viewport, + width: slice.width(), + height: slice.height(), + }; + ReactDOM.render( + , + document.getElementById(slice.containerId), + ); +} +module.exports = deckHex; diff --git a/superset/assets/visualizations/deckgl/scatter.jsx b/superset/assets/visualizations/deckgl/scatter.jsx new file mode 100644 index 0000000000000..007dc9007c09c --- /dev/null +++ b/superset/assets/visualizations/deckgl/scatter.jsx @@ -0,0 +1,56 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { ScatterplotLayer } from 'deck.gl'; + +import DeckGLContainer from './DeckGLContainer'; +import { getColorFromScheme, hexToRGB } from '../../javascripts/modules/colors'; +import { unitToRadius } from '../../javascripts/modules/geo'; + +function deckScatter(slice, payload, setControlValue) { + const fd = slice.formData; + const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 }; + const fixedColor = [c.r, c.g, c.b, 255 * c.a]; + + const data = payload.data.features.map((d) => { + let radius = unitToRadius(fd.point_unit, d.radius) || 10; + if (fd.multiplier) { + radius *= fd.multiplier; + } + let color; + if (fd.dimension) { + color = hexToRGB(getColorFromScheme(d.cat_color, fd.color_scheme), c.a * 255); + } else { + color = fixedColor; + } + return { + ...d, + radius, + color, + }; + }); + + const layer = new ScatterplotLayer({ + id: 'scatterplot-layer', + data, + pickable: true, + fp64: true, + // onHover: info => console.log('Hovered:', info), + outline: false, + }); + const viewport = { + ...fd.viewport, + width: slice.width(), + height: slice.height(), + }; + ReactDOM.render( + , + document.getElementById(slice.containerId), + ); +} +module.exports = deckScatter; diff --git a/superset/assets/visualizations/deckgl/screengrid.jsx b/superset/assets/visualizations/deckgl/screengrid.jsx new file mode 100644 index 0000000000000..e994a450f1516 --- /dev/null +++ b/superset/assets/visualizations/deckgl/screengrid.jsx @@ -0,0 +1,43 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { ScreenGridLayer } from 'deck.gl'; + +import DeckGLContainer from './DeckGLContainer'; + +function deckScreenGridLayer(slice, payload, setControlValue) { + const fd = slice.formData; + const c = fd.color_picker; + const data = payload.data.features.map(d => ({ + ...d, + color: [c.r, c.g, c.b, 255 * c.a], + })); + + const viewport = { + ...fd.viewport, + width: slice.width(), + height: slice.height(), + }; + // Passing a layer creator function instead of a layer since the + // layer needs to be regenerated at each render + const layer = () => new ScreenGridLayer({ + id: 'screengrid-layer', + data, + pickable: true, + cellSizePixels: fd.grid_size, + minColor: [c.r, c.g, c.b, 0], + maxColor: [c.r, c.g, c.b, 255 * c.a], + outline: false, + getWeight: d => d.weight || 0, + }); + ReactDOM.render( + , + document.getElementById(slice.containerId), + ); +} +module.exports = deckScreenGridLayer; diff --git a/superset/assets/visualizations/main.js b/superset/assets/visualizations/main.js index 78e81ab6d7340..2afc57b6175ea 100644 --- a/superset/assets/visualizations/main.js +++ b/superset/assets/visualizations/main.js @@ -36,5 +36,9 @@ const vizMap = { event_flow: require('./EventFlow.jsx'), paired_ttest: require('./paired_ttest.jsx'), partition: require('./partition.js'), + deck_scatter: require('./deckgl/scatter.jsx'), + deck_screengrid: require('./deckgl/screengrid.jsx'), + deck_grid: require('./deckgl/grid.jsx'), + deck_hex: require('./deckgl/hex.jsx'), }; export default vizMap; diff --git a/superset/cli.py b/superset/cli.py index 0c666e1261c0e..e6dde29d2563d 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -133,10 +133,16 @@ def load_examples(load_test_data): print("Loading [Misc Charts] dashboard") data.load_misc_dashboard() + print("Loading DECK.gl demo") + data.load_deck_dash() + if load_test_data: print("Loading [Unicode test data]") data.load_unicode_test_data() + print("Loading flights data") + data.load_flights() + @manager.option( '-d', '--datasource', diff --git a/superset/data/__init__.py b/superset/data/__init__.py index 39151062f27a0..a91d66cab6d83 100644 --- a/superset/data/__init__.py +++ b/superset/data/__init__.py @@ -1227,3 +1227,276 @@ def load_misc_dashboard(): dash.slices = slices db.session.merge(dash) db.session.commit() + + +def load_deck_dash(): + print("Loading deck.gl dashboard") + slices = [] + tbl = db.session.query(TBL).filter_by(table_name='long_lat').first() + slice_data = { + "all_columns_x": "LON", + "all_columns_y": "LAT", + "color_picker": { + "r": 205, + "g": 0, + "b": 3, + "a": 0.82, + }, + "datasource": "5__table", + "filters": [], + "granularity_sqla": "date", + "groupby": [], + "having": "", + "mapbox_style": "mapbox://styles/mapbox/light-v9", + "multiplier": 10, + "point_radius_fixed": {"type": "metric", "value": "count"}, + "point_unit": "square_m", + "row_limit": 5000, + "since": "2014-01-01", + "size": "count", + "time_grain_sqla": "Time Column", + "until": "now", + "viewport": { + "bearing": -4.952916738791771, + "latitude": 37.78926922909199, + "longitude": -122.42613341901688, + "pitch": 4.750411100577438, + "zoom": 12.729132798697304, + }, + "viz_type": "deck_scatter", + "where": "", + } + + print("Creating Scatterplot slice") + slc = Slice( + slice_name="Scatterplot", + viz_type='deck_scatter', + datasource_type='table', + datasource_id=tbl.id, + params=get_slice_json(slice_data), + ) + merge_slice(slc) + slices.append(slc) + + slice_data = { + "point_unit": "square_m", + "filters": [], + "row_limit": 5000, + "all_columns_y": "LAT", + "all_columns_x": "LON", + "mapbox_style": "mapbox://styles/mapbox/dark-v9", + "granularity_sqla": "date", + "size": "count", + "viz_type": "deck_screengrid", + "since": "2014-01-01", + "point_radius": "Auto", + "until": "now", + "color_picker": {"a": 1, + "r": 14, + "b": 0, + "g": 255}, + "grid_size": 20, + "where": "", + "having": "", + "viewport": { + "zoom": 14.161641703941438, + "longitude": -122.41827069521386, + "bearing": -4.952916738791771, + "latitude": 37.76024135844065, + "pitch": 4.750411100577438, + }, + "point_radius_fixed": {"type": "fix", "value": 2000}, + "datasource": "5__table", + "time_grain_sqla": "Time Column", + "groupby": [], + } + print("Creating Screen Grid slice") + slc = Slice( + slice_name="Screen grid", + viz_type='deck_screengrid', + datasource_type='table', + datasource_id=tbl.id, + params=get_slice_json(slice_data), + ) + merge_slice(slc) + slices.append(slc) + + slice_data = { + "filters": [], + "row_limit": 5000, + "all_columns_y": "LAT", + "all_columns_x": "LON", + "mapbox_style": "mapbox://styles/mapbox/streets-v9", + "granularity_sqla": "date", + "size": "count", + "viz_type": "deck_hex", + "since": "2014-01-01", + "point_radius_unit": "Pixels", + "point_radius": "Auto", + "until": "now", + "color_picker": { + "a": 1, + "r": 14, + "b": 0, + "g": 255, + }, + "grid_size": 40, + "extruded": True, + "having": "", + "viewport": { + "latitude": 37.789795085160335, + "pitch": 54.08961642447763, + "zoom": 13.835465702403654, + "longitude": -122.40632230075536, + "bearing": -2.3984797349335167, + }, + "where": "", + "point_radius_fixed": {"type": "fix", "value": 2000}, + "datasource": "5__table", + "time_grain_sqla": "Time Column", + "groupby": [], + } + print("Creating Hex slice") + slc = Slice( + slice_name="Hexagons", + viz_type='deck_hex', + datasource_type='table', + datasource_id=tbl.id, + params=get_slice_json(slice_data), + ) + merge_slice(slc) + slices.append(slc) + + slice_data = { + "filters": [], + "row_limit": 5000, + "all_columns_y": "LAT", + "all_columns_x": "LON", + "mapbox_style": "mapbox://styles/mapbox/satellite-streets-v9", + "granularity_sqla": "date", + "size": "count", + "viz_type": "deck_grid", + "since": "2014-01-01", + "point_radius_unit": "Pixels", + "point_radius": "Auto", + "until": "now", + "color_picker": { + "a": 1, + "r": 14, + "b": 0, + "g": 255, + }, + "grid_size": 120, + "extruded": True, + "having": "", + "viewport": { + "longitude": -122.42066918995666, + "bearing": 155.80099696026355, + "zoom": 12.699690845482069, + "latitude": 37.7942314882596, + "pitch": 53.470800300695146, + }, + "where": "", + "point_radius_fixed": {"type": "fix", "value": 2000}, + "datasource": "5__table", + "time_grain_sqla": "Time Column", + "groupby": [], + } + print("Creating Grid slice") + slc = Slice( + slice_name="Grid", + viz_type='deck_grid', + datasource_type='table', + datasource_id=tbl.id, + params=get_slice_json(slice_data), + ) + merge_slice(slc) + slices.append(slc) + + print("Creating a dashboard") + title = "deck.gl Demo" + dash = db.session.query(Dash).filter_by(dashboard_title=title).first() + + if not dash: + dash = Dash() + js = textwrap.dedent("""\ + [ + { + "col": 1, + "row": 0, + "size_x": 6, + "size_y": 4, + "slice_id": "37" + }, + { + "col": 7, + "row": 0, + "size_x": 6, + "size_y": 4, + "slice_id": "38" + }, + { + "col": 7, + "row": 4, + "size_x": 6, + "size_y": 4, + "slice_id": "39" + }, + { + "col": 1, + "row": 4, + "size_x": 6, + "size_y": 4, + "slice_id": "40" + } + ] + """) + l = json.loads(js) + for i, pos in enumerate(l): + pos['slice_id'] = str(slices[i].id) + dash.dashboard_title = title + dash.position_json = json.dumps(l, indent=4) + dash.slug = "deck" + dash.slices = slices + db.session.merge(dash) + db.session.commit() + + +def load_flights(): + """Loading random time series data from a zip file in the repo""" + with gzip.open(os.path.join(DATA_FOLDER, 'fligth_data.csv.gz')) as f: + pdf = pd.read_csv(f, encoding='latin-1') + + # Loading airports info to join and get lat/long + with gzip.open(os.path.join(DATA_FOLDER, 'airports.csv.gz')) as f: + airports = pd.read_csv(f, encoding='latin-1') + airports = airports.set_index('IATA_CODE') + + pdf['ds'] = pdf.YEAR.map(str) + '-0' + pdf.MONTH.map(str) + '-0' + pdf.DAY.map(str) + pdf.ds = pd.to_datetime(pdf.ds) + del pdf['YEAR'] + del pdf['MONTH'] + del pdf['DAY'] + + pdf = pdf.join(airports, on='ORIGIN_AIRPORT', rsuffix='_ORIG') + pdf = pdf.join(airports, on='DESTINATION_AIRPORT', rsuffix='_DEST') + pdf.to_sql( + 'flights', + db.engine, + if_exists='replace', + chunksize=500, + dtype={ + 'ds': DateTime, + }, + index=False) + print("Done loading table!") + + print("Creating table [random_time_series] reference") + obj = db.session.query(TBL).filter_by(table_name='random_time_series').first() + if not obj: + obj = TBL(table_name='flights') + obj.main_dttm_col = 'ds' + obj.database = get_or_create_main_db() + db.session.merge(obj) + db.session.commit() + obj.fetch_metadata() diff --git a/superset/data/airports.csv.gz b/superset/data/airports.csv.gz new file mode 100644 index 0000000000000..3043486664476 Binary files /dev/null and b/superset/data/airports.csv.gz differ diff --git a/superset/data/fligth_data.csv.gz b/superset/data/fligth_data.csv.gz new file mode 100644 index 0000000000000..bbdebdfafcba7 Binary files /dev/null and b/superset/data/fligth_data.csv.gz differ diff --git a/superset/viz.py b/superset/viz.py index 025e9c52b0c52..3b44bce328ac3 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -1731,7 +1731,116 @@ def get_data(self, df): } +class BaseDeckGLViz(BaseViz): + + """Base class for deck.gl visualizations""" + + viz_type = "mapbox" + verbose_name = _("Mapbox") + is_timeseries = False + credits = 'deck.gl' + + def get_metrics(self): + self.metric = self.form_data.get('size') + return [self.metric] + + def get_properties(self, d): + return { + 'weight': d.get(self.metric) or 1, + } + + def get_position(self, d): + return [ + d.get(self.form_data.get('all_columns_x')), + d.get(self.form_data.get('all_columns_y')), + ] + + def query_obj(self): + d = super(BaseDeckGLViz, self).query_obj() + fd = self.form_data + + d['groupby'] = [fd.get('all_columns_x'), fd.get('all_columns_y')] + if fd.get('dimension'): + d['groupby'] += [fd.get('dimension')] + + d['metrics'] = self.get_metrics() + return d + + def get_data(self, df): + fd = self.form_data + + # using geoJSON formatting + features = [] + for d in df.to_dict(orient='records'): + d = dict(position=self.get_position(d), **self.get_properties(d)) + features.append(d) + return { + "features": features, + "mapboxApiKey": config.get('MAPBOX_API_KEY'), + } + + +class DeckScatterViz(BaseDeckGLViz): + + """deck.gl's ScatterLayer""" + + viz_type = "deck_scatter" + verbose_name = _("Deck.gl - Scatter plot") + + def query_obj(self): + self.point_radius_fixed = self.form_data.get('point_radius_fixed') + return super(DeckScatterViz, self).query_obj() + + def get_metrics(self): + if self.point_radius_fixed.get('type') == 'metric': + self.metric = self.point_radius_fixed.get('value') + else: + self.metric = 'count' + return [self.metric] + + def get_properties(self, d): + return { + "radius": self.fixed_value if self.fixed_value else d.get(self.metric), + "cat_color": d.get(self.dim) if self.dim else None, + } + + def get_data(self, df): + fd = self.form_data + self.point_radius_fixed = fd.get('point_radius_fixed') + self.fixed_value = None + self.dim = self.form_data.get('dimension') + if self.point_radius_fixed.get('type') != 'metric': + self.fixed_value = self.point_radius_fixed.get('value') + + return super(DeckScatterViz, self).get_data(df) + + +class DeckScreengrid(BaseDeckGLViz): + + """deck.gl's ScreenGridLayer""" + + viz_type = "deck_screengrid" + verbose_name = _("Deck.gl - Screen Grid") + + +class DeckGrid(BaseDeckGLViz): + + """deck.gl's DeckLayer""" + + viz_type = "deck_grid" + verbose_name = _("Deck.gl - 3D Grid") + + +class DeckHex(BaseDeckGLViz): + + """deck.gl's DeckLayer""" + + viz_type = "deck_hex" + verbose_name = _("Deck.gl - 3D HEX") + + class EventFlowViz(BaseViz): + """A visualization to explore patterns in event sequences""" viz_type = "event_flow"