From da20a443783242f571eee1231fc843ce3fb6836e Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 4 Jan 2022 08:37:44 +0200 Subject: [PATCH 01/20] feat(plugin-chart-echarts): support non-timeseries x-axis --- .../src/operators/pivotOperator.ts | 6 ++- .../src/utils/featureFlags.ts | 1 + .../src/Timeseries/Area/controlPanel.tsx | 4 +- .../src/Timeseries/Area/index.ts | 12 +---- .../src/Timeseries/EchartsTimeseries.tsx | 4 +- .../src/Timeseries/Regular/Bar/index.ts | 20 +++++--- .../src/Timeseries/Regular/Line/index.ts | 22 ++++++--- .../src/Timeseries/Regular/Scatter/index.ts | 22 ++++++--- .../Timeseries/Regular/SmoothLine/index.ts | 22 ++++++--- .../src/Timeseries/Regular/controlPanel.tsx | 8 ++-- .../src/Timeseries/Step/controlPanel.tsx | 4 +- .../src/Timeseries/Step/index.ts | 22 ++++++--- .../src/Timeseries/buildQuery.ts | 10 +++- .../src/Timeseries/controlPanel.tsx | 4 +- .../src/Timeseries/index.ts | 32 ++++++------- .../src/Timeseries/transformProps.ts | 41 +++++++++++++--- .../plugin-chart-echarts/src/controls.tsx | 18 +++++++ .../plugin-chart-echarts/src/utils/prophet.ts | 10 ++-- .../plugin-chart-echarts/src/utils/series.ts | 47 +++++++++++-------- 19 files changed, 206 insertions(+), 103 deletions(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts index 8789efb44e9ca..abae4563ec1fd 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts @@ -30,7 +30,9 @@ export const pivotOperator: PostProcessingFactory< PostProcessingPivot | undefined > = (formData, queryObject) => { const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel); - if (queryObject.is_timeseries && metricLabels.length) { + const { x_axis } = formData; + const index = !x_axis || queryObject.is_timeseries ? TIME_COLUMN : x_axis; + if (index && metricLabels.length) { if (isValidTimeCompare(formData, queryObject)) { return timeComparePivotOperator(formData, queryObject); } @@ -38,7 +40,7 @@ export const pivotOperator: PostProcessingFactory< return { operation: 'pivot', options: { - index: [TIME_COLUMN], + index: [index], columns: ensureIsArray(queryObject.columns).map(getColumnLabel), // Create 'dummy' mean aggregates to assign cell values in pivot table // use the 'mean' aggregates to avoid drop NaN. PR: https://github.com/apache-superset/superset-ui/pull/1231 diff --git a/superset-frontend/packages/superset-ui-core/src/utils/featureFlags.ts b/superset-frontend/packages/superset-ui-core/src/utils/featureFlags.ts index 86c69ea2b7858..54bd9b5f65127 100644 --- a/superset-frontend/packages/superset-ui-core/src/utils/featureFlags.ts +++ b/superset-frontend/packages/superset-ui-core/src/utils/featureFlags.ts @@ -51,6 +51,7 @@ export enum FeatureFlag { ALERTS_ATTACH_REPORTS = 'ALERTS_ATTACH_REPORTS', ALLOW_FULL_CSV_EXPORT = 'ALLOW_FULL_CSV_EXPORT', UX_BETA = 'UX_BETA', + GENERIC_CHART_AXES = 'GENERIC_CHART_AXES', } export type ScheduleQueriesProps = { JSONSCHEMA: { diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/controlPanel.tsx index 0dcc76fcc8df2..720eb9428723d 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -36,6 +36,7 @@ import { legendSection, richTooltipSection, showValueSection, + xAxisControl, } from '../../controls'; const { @@ -59,6 +60,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts index e0c0af270686c..9e664c280e8b3 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts @@ -43,16 +43,6 @@ export default class EchartsAreaChartPlugin extends ChartPlugin< EchartsTimeseriesFormData, EchartsTimeseriesChartProps > { - /** - * The constructor is used to pass relevant metadata and callbacks that get - * registered in respective registries that are used throughout the library - * and application. A more thorough description of each property is given in - * the respective imported file. - * - * It is worth noting that `buildQuery` and is optional, and only needed for - * advanced visualizations that require either post processing operations - * (pivoting, rolling aggregations, sorting etc) or submitting multiple queries. - */ constructor() { super({ buildQuery, @@ -63,7 +53,7 @@ export default class EchartsAreaChartPlugin extends ChartPlugin< category: t('Evolution'), credits: ['https://echarts.apache.org'], description: t( - 'Time-series Area chart are similar to line chart in that they represent variables with the same scale, but area charts stack the metrics on top of each other. An area chart in Superset can be stream, stack, or expand.', + 'Area charts are similar to line charts in that they represent variables with the same scale, but area charts stack the metrics on top of each other.', ), exampleGallery: [{ url: example1 }], supportedAnnotationTypes: [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx index 2bf103e2bd9e4..e68fa88693aaf 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx @@ -130,7 +130,9 @@ export default function EchartsTimeseries({ } // Ensure that double-click events do not trigger single click event. So we put it in the timer. clickTimer.current = setTimeout(() => { - const { seriesName: name } = props; + const { seriesName: name, value } = props; + const xValue = value[0].getTime?.() || value[0]; + console.log(xValue); const values = Object.values(selectedValues); if (values.includes(name)) { handleChange(values.filter(v => v !== name)); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/index.ts index 0c23a19bd6fef..0ffc09098c70b 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from '../../buildQuery'; import controlPanel from './controlPanel'; @@ -58,9 +60,11 @@ export default class EchartsTimeseriesBarChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Time-series Bar Charts are used to show the changes in a metric over time as a series of bars.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Bar Charts are used to show metrics as a series of bars.') + : t( + 'Time-series Bar Charts are used to show the changes in a metric over time as a series of bars.', + ), exampleGallery: [ { url: example1 }, { url: example2 }, @@ -72,7 +76,9 @@ export default class EchartsTimeseriesBarChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Bar Chart v2'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Bar Chart v2') + : t('Time-series Bar Chart v2'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Line/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Line/index.ts index 0ee78dae1045f..6f4a780c36fc3 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Line/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Line/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from '../../buildQuery'; import controlPanel from '../controlPanel'; @@ -57,9 +59,13 @@ export default class EchartsTimeseriesLineChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Time-series line chart is used to visualize repeated measurements taken over regular time intervals. Line chart is a type of chart which displays information as a series of data points connected by straight line segments. It is a basic type of chart common in many fields.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Line chart is used to visualize measurements taken over a given category. Line chart is a type of chart which displays information as a series of data points connected by straight line segments. It is a basic type of chart common in many fields.', + ) + : t( + 'Time-series line chart is used to visualize repeated measurements taken over regular time intervals. Line chart is a type of chart which displays information as a series of data points connected by straight line segments. It is a basic type of chart common in many fields.', + ), exampleGallery: [{ url: example1 }, { url: example2 }], supportedAnnotationTypes: [ AnnotationType.Event, @@ -67,7 +73,9 @@ export default class EchartsTimeseriesLineChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Line Chart'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Line Chart') + : t('Time-series Line Chart'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/index.ts index 98a7898932d14..7c77868a58451 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from '../../buildQuery'; import controlPanel from './controlPanel'; @@ -56,9 +58,13 @@ export default class EchartsTimeseriesScatterChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Time-series Scatter Plot has time on the horizontal axis in linear units, and the points are connected in order. It shows a statistical relationship between two variables.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Scatter Plot has the horizontal axis in linear units, and the points are connected in order. It shows a statistical relationship between two variables.', + ) + : t( + 'Time-series Scatter Plot has time on the horizontal axis in linear units, and the points are connected in order. It shows a statistical relationship between two variables.', + ), exampleGallery: [{ url: example1 }], supportedAnnotationTypes: [ AnnotationType.Event, @@ -66,7 +72,9 @@ export default class EchartsTimeseriesScatterChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Scatter Plot'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Scatter Plot') + : t('Time-series Scatter Plot'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/SmoothLine/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/SmoothLine/index.ts index db592e1b80c5d..ee348b272cd45 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/SmoothLine/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/SmoothLine/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from '../../buildQuery'; import controlPanel from '../controlPanel'; @@ -56,9 +58,13 @@ export default class EchartsTimeseriesSmoothLineChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Time-series Smooth-line is a variation of line chart. Without angles and hard edges, Smooth-line looks more smarter and more professional.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Smooth-line is a variation of the line chart. Without angles and hard edges, Smooth-line sometimes looks smarter and more professional.', + ) + : t( + 'Time-series Smooth-line is a variation of the line chart. Without angles and hard edges, Smooth-line sometimes looks smarter and more professional.', + ), exampleGallery: [{ url: example1 }], supportedAnnotationTypes: [ AnnotationType.Event, @@ -66,7 +72,9 @@ export default class EchartsTimeseriesSmoothLineChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Smooth Line'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Smooth Line') + : t('Time-series Smooth Line'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/controlPanel.tsx index 3e0dc46b49e65..14edc341f2bab 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -31,7 +31,8 @@ import { DEFAULT_FORM_DATA, EchartsTimeseriesContributionType } from '../types'; import { legendSection, richTooltipSection, - showValueSection, + showValueSectionWithoutStack, + xAxisControl, } from '../../controls'; const { @@ -53,6 +54,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], [ @@ -100,7 +102,7 @@ const config: ControlPanelConfig = { expanded: true, controlSetRows: [ ['color_scheme'], - ...showValueSection, + ...showValueSectionWithoutStack, [ { name: 'markerEnabled', diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/controlPanel.tsx index 1deb723795d48..8178e4929782a 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -36,6 +36,7 @@ import { legendSection, richTooltipSection, showValueSection, + xAxisControl, } from '../../controls'; const { @@ -59,6 +60,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/index.ts index ddf18ebe5b369..2a24b708419ef 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Step/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from '../buildQuery'; import controlPanel from './controlPanel'; @@ -47,9 +49,13 @@ export default class EchartsTimeseriesStepChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Time-series Stepped-line graph (also called step chart) is a variation of line chart but with the line forming a series of steps between data points. A step chart can be useful when you want to show the changes that occur at irregular intervals.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Stepped-line graph (also called step chart) is a variation of line chart but with the line forming a series of steps between data points. A step chart can be useful when you want to show the changes that occur at irregular intervals.', + ) + : t( + 'Time-series Stepped-line graph (also called step chart) is a variation of line chart but with the line forming a series of steps between data points. A step chart can be useful when you want to show the changes that occur at irregular intervals.', + ), exampleGallery: [{ url: example1 }, { url: example2 }], supportedAnnotationTypes: [ AnnotationType.Event, @@ -57,7 +63,9 @@ export default class EchartsTimeseriesStepChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Stepped Line'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Stepped Line') + : t('Time-series Stepped Line'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts index 5d02f2fb0fc5c..5a8e190010ff8 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts @@ -18,6 +18,8 @@ */ import { buildQueryContext, + DTTM_ALIAS, + ensureIsArray, QueryFormData, normalizeOrderBy, RollingType, @@ -35,11 +37,13 @@ import { } from '@superset-ui/chart-controls'; export default function buildQuery(formData: QueryFormData) { + const { x_axis, groupby } = formData; + const is_timeseries = x_axis === DTTM_ALIAS || !x_axis; return buildQueryContext(formData, baseQueryObject => { const pivotOperatorInRuntime: PostProcessingPivot | undefined = pivotOperator(formData, { ...baseQueryObject, - is_timeseries: true, + index: x_axis, }); if ( pivotOperatorInRuntime && @@ -57,7 +61,9 @@ export default function buildQuery(formData: QueryFormData) { return [ { ...baseQueryObject, - is_timeseries: true, + columns: [...ensureIsArray(x_axis), ...ensureIsArray(groupby)], + series_columns: groupby, + is_timeseries, // todo: move `normalizeOrderBy to extractQueryFields` orderby: normalizeOrderBy(baseQueryObject).orderby, time_offsets: isValidTimeCompare(formData, baseQueryObject) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/controlPanel.tsx index 5878882d3e4cf..1012b2f6e7df2 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -36,6 +36,7 @@ import { legendSection, richTooltipSection, showValueSection, + xAxisControl, } from '../controls'; const { @@ -60,6 +61,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/index.ts index 48856ab5a32ba..062d741402ff7 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/index.ts @@ -17,11 +17,13 @@ * under the License. */ import { - t, - ChartMetadata, - ChartPlugin, AnnotationType, Behavior, + ChartMetadata, + ChartPlugin, + FeatureFlag, + isFeatureEnabled, + t, } from '@superset-ui/core'; import buildQuery from './buildQuery'; import controlPanel from './controlPanel'; @@ -37,16 +39,6 @@ export default class EchartsTimeseriesChartPlugin extends ChartPlugin< EchartsTimeseriesFormData, EchartsTimeseriesChartProps > { - /** - * The constructor is used to pass relevant metadata and callbacks that get - * registered in respective registries that are used throughout the library - * and application. A more thorough description of each property is given in - * the respective imported file. - * - * It is worth noting that `buildQuery` and is optional, and only needed for - * advanced visualizations that require either post processing operations - * (pivoting, rolling aggregations, sorting etc) or submitting multiple queries. - */ constructor() { super({ buildQuery, @@ -56,9 +48,13 @@ export default class EchartsTimeseriesChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Swiss army knife for visualizing time series data. Choose between step, line, scatter, and bar charts. This viz type has many customization options as well.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Swiss army knife for visualizing data. Choose between step, line, scatter, and bar charts. This viz type has many customization options as well.', + ) + : t( + 'Swiss army knife for visualizing time series data. Choose between step, line, scatter, and bar charts. This viz type has many customization options as well.', + ), exampleGallery: [{ url: example }], supportedAnnotationTypes: [ AnnotationType.Event, @@ -66,7 +62,9 @@ export default class EchartsTimeseriesChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Chart'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Generic Chart') + : t('Time-series Chart'), tags: [ t('Advanced-Analytics'), t('Aesthetic'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts index b63feb35737c5..f6190904a489d 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts @@ -20,28 +20,32 @@ import { AnnotationLayer, CategoricalColorNamespace, + DataRecordValue, + DTTM_ALIAS, + GenericDataType, getNumberFormatter, isEventAnnotationLayer, isFormulaAnnotationLayer, isIntervalAnnotationLayer, isTimeseriesAnnotationLayer, TimeseriesChartDataResponseResult, - DataRecordValue, } from '@superset-ui/core'; import { EChartsCoreOption, SeriesOption } from 'echarts'; import { DEFAULT_FORM_DATA, EchartsTimeseriesChartProps, EchartsTimeseriesFormData, + EchartsTimeseriesSeriesType, TimeseriesChartTransformedProps, } from './types'; import { ForecastSeriesEnum, ProphetValue } from '../types'; import { parseYAxisBound } from '../utils/controls'; import { + currentSeries, dedupSeries, extractTimeseriesSeries, + getColtypesMapping, getLegendProps, - currentSeries, } from '../utils/series'; import { extractAnnotationLabels } from '../utils/annotation'; import { @@ -77,8 +81,10 @@ export default function transformProps( datasource, } = chartProps; const { verboseMap = {} } = datasource; + const [queryData] = queriesData; const { annotation_data: annotationData_, data = [] } = - queriesData[0] as TimeseriesChartDataResponseResult; + queryData as TimeseriesChartDataResponseResult; + const dataTypes = getColtypesMapping(queryData); const annotationData = annotationData_ || {}; const { @@ -106,6 +112,7 @@ export default function transformProps( tooltipSortByMetric, zoomable, richTooltip, + xAxis: xAxisOrig, xAxisLabelRotation, emitFilter, groupby, @@ -119,12 +126,31 @@ export default function transformProps( }: EchartsTimeseriesFormData = { ...DEFAULT_FORM_DATA, ...formData }; const colorScale = CategoricalColorNamespace.getScale(colorScheme as string); const rebasedData = rebaseTimeseriesDatum(data, verboseMap); + const xAxisCol = xAxisOrig || DTTM_ALIAS; const rawSeries = extractTimeseriesSeries(rebasedData, { fillNeighborValue: stack && !forecastEnabled ? 0 : undefined, + xAxis: xAxisCol, + removeNulls: [ + EchartsTimeseriesSeriesType.Line, + EchartsTimeseriesSeriesType.Scatter, + ].includes(seriesType), }); const seriesContexts = extractForecastSeriesContexts( Object.values(rawSeries).map(series => series.name as string), ); + const xAxisDataType = dataTypes?.[xAxisCol]; + let xAxisType: 'time' | 'value' | 'category'; + switch (xAxisDataType) { + case GenericDataType.TEMPORAL: + xAxisType = 'time'; + break; + case GenericDataType.NUMERIC: + xAxisType = 'value'; + break; + default: + xAxisType = 'category'; + break; + } const series: SeriesOption[] = []; const formatter = getNumberFormatter(contributionMode ? ',.0%' : yAxisFormat); @@ -133,7 +159,7 @@ export default function transformProps( rebasedData.forEach(data => { const values = Object.keys(data).reduce((prev, curr) => { - if (curr === '__timestamp') { + if (curr === DTTM_ALIAS) { return prev; } const value = data[curr] || 0; @@ -223,7 +249,10 @@ export default function transformProps( if (max === undefined) max = 1; } - const tooltipFormatter = getTooltipTimeFormatter(tooltipTimeFormat); + const tooltipFormatter = + xAxisDataType === GenericDataType.TEMPORAL + ? getTooltipTimeFormatter(tooltipTimeFormat) + : String; const xAxisFormatter = getXAxisFormatter(xAxisTimeFormat); const labelMap = series.reduce( @@ -269,7 +298,7 @@ export default function transformProps( ...padding, }, xAxis: { - type: 'time', + type: xAxisType, name: xAxisTitle, nameGap: xAxisTitleMargin, nameLocation: 'middle', diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx index 76c7c42186ffc..978367b54a657 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx @@ -136,12 +136,30 @@ const onlyTotalControl = { }, }; +export const xAxisControl = { + name: 'x_axis', + config: { + ...sharedControls.groupby, + label: t('X-axis'), + default: null, + multi: false, + description: t( + 'Dimension to use on x-axis. Leave blank to default to temporal column.', + ), + }, +}; + export const showValueSection = [ [showValueControl], [stackControl], [onlyTotalControl], ]; +export const showValueSectionWithoutStack = [ + [showValueControl], + [onlyTotalControl], +]; + const richTooltipControl = { name: 'rich_tooltip', config: { diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/prophet.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/prophet.ts index 2f8acf30602b9..82a747e53718c 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/utils/prophet.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/prophet.ts @@ -16,7 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -import { TimeseriesDataRecord, NumberFormatter } from '@superset-ui/core'; +import { + TimeseriesDataRecord, + NumberFormatter, + DTTM_ALIAS, +} from '@superset-ui/core'; import { CallbackDataParams, OptionName } from 'echarts/types/src/util/types'; import { TooltipMarker } from 'echarts/types/src/util/format'; import { @@ -117,7 +121,7 @@ export function rebaseTimeseriesDatum( // eslint-disable-next-line @typescript-eslint/no-unsafe-return return data.map(row => { - const newRow: TimeseriesDataRecord = { __timestamp: '' }; + const newRow: TimeseriesDataRecord = { [DTTM_ALIAS]: '' }; keys.forEach(key => { const forecastContext = extractForecastSeriesContext(key); const lowerKey = `${forecastContext.name}${ForecastSeriesEnum.ForecastLower}`; @@ -131,7 +135,7 @@ export function rebaseTimeseriesDatum( value -= row[lowerKey] as number; } const newKey = - key !== '__timestamp' && verboseMap[key] ? verboseMap[key] : key; + key !== DTTM_ALIAS && verboseMap[key] ? verboseMap[key] : key; newRow[newKey] = value; }); // eslint-disable-next-line @typescript-eslint/no-unsafe-return diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts index 324ab06d10884..88c4776568bff 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts @@ -21,6 +21,7 @@ import { ChartDataResponseResult, DataRecord, DataRecordValue, + DTTM_ALIAS, ensureIsArray, GenericDataType, NumberFormatter, @@ -38,35 +39,38 @@ function isDefined(value: T | undefined | null): boolean { export function extractTimeseriesSeries( data: TimeseriesDataRecord[], - opts: { fillNeighborValue?: number } = {}, + opts: { + fillNeighborValue?: number; + xAxis?: string; + removeNulls?: boolean; + } = {}, ): SeriesOption[] { - const { fillNeighborValue } = opts; + const { fillNeighborValue, xAxis = DTTM_ALIAS, removeNulls = false } = opts; if (data.length === 0) return []; const rows: TimeseriesDataRecord[] = data.map(datum => ({ ...datum, - __timestamp: - datum.__timestamp || datum.__timestamp === 0 - ? new Date(datum.__timestamp) - : null, + [xAxis]: datum[xAxis], })); return Object.keys(rows[0]) - .filter(key => key !== '__timestamp') + .filter(key => key !== xAxis && key !== DTTM_ALIAS) .map(key => ({ id: key, name: key, - data: rows.map((row, idx) => { - const isNextToDefinedValue = - isDefined(rows[idx - 1]?.[key]) || isDefined(rows[idx + 1]?.[key]); - return [ - row.__timestamp, - !isDefined(row[key]) && - isNextToDefinedValue && - fillNeighborValue !== undefined - ? fillNeighborValue - : row[key], - ]; - }), + data: rows + .map((row, idx) => { + const isNextToDefinedValue = + isDefined(rows[idx - 1]?.[key]) || isDefined(rows[idx + 1]?.[key]); + return [ + row[xAxis], + !isDefined(row[key]) && + isNextToDefinedValue && + fillNeighborValue !== undefined + ? fillNeighborValue + : row[key], + ]; + }) + .filter(row => !removeNulls || row[1] !== null), })); } @@ -102,7 +106,10 @@ export function formatSeriesName( export const getColtypesMapping = ({ coltypes = [], colnames = [], -}: ChartDataResponseResult): Record => +}: Pick): Record< + string, + GenericDataType +> => colnames.reduce( (accumulator, item, index) => ({ ...accumulator, [item]: coltypes[index] }), {}, From 2efbac1e75f9b9d01f87f1a6b4c0decf52e70670 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 4 Jan 2022 11:49:14 +0200 Subject: [PATCH 02/20] fix tests --- .../src/operators/pivotOperator.ts | 6 ++-- .../src/query/types/PostProcessing.ts | 10 ++++-- .../plugin-chart-echarts/src/utils/series.ts | 2 +- .../test/Timeseries/transformProps.test.ts | 24 ++++++++------ .../test/utils/annotation.test.ts | 6 ++++ .../test/utils/series.test.ts | 32 +++++++++---------- 6 files changed, 48 insertions(+), 32 deletions(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts index abae4563ec1fd..629fc18cacf90 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts @@ -30,9 +30,9 @@ export const pivotOperator: PostProcessingFactory< PostProcessingPivot | undefined > = (formData, queryObject) => { const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel); - const { x_axis } = formData; - const index = !x_axis || queryObject.is_timeseries ? TIME_COLUMN : x_axis; - if (index && metricLabels.length) { + const { x_axis: xAxis } = formData; + if ((xAxis || queryObject.is_timeseries) && metricLabels.length) { + const index = !xAxis || queryObject.is_timeseries ? TIME_COLUMN : xAxis; if (isValidTimeCompare(formData, queryObject)) { return timeComparePivotOperator(formData, queryObject); } diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts b/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts index f38ed411f1fa5..6fab68ad077a7 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/PostProcessing.ts @@ -94,10 +94,16 @@ export interface PostProcessingContribution { export interface PostProcessingPivot { operation: 'pivot'; options: { - index: string[]; - columns: string[]; aggregates: Aggregates; + column_fill_value?: string; + columns: string[]; + combine_value_with_metric?: boolean; + drop_missing_columns?: boolean; flatten_columns?: boolean; + index: string[]; + marginal_distribution_name?: string; + marginal_distributions?: boolean; + metric_fill_value?: any; reset_index?: boolean; }; } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts index 88c4776568bff..763713f520733 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts @@ -126,7 +126,7 @@ export function extractGroupbyLabel({ groupby?: string[] | null; numberFormatter?: NumberFormatter; timeFormatter?: TimeFormatter; - coltypeMapping: Record; + coltypeMapping?: Record; }): string { return ensureIsArray(groupby) .map(val => diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts index 96c5865d76c66..572675dfe798a 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts @@ -17,6 +17,8 @@ * under the License. */ import { + AnnotationStyle, + AnnotationType, ChartProps, EventAnnotationLayer, FormulaAnnotationLayer, @@ -32,6 +34,7 @@ describe('EchartsTimeseries tranformProps', () => { granularity_sqla: 'ds', metric: 'sum__num', groupby: ['foo', 'bar'], + viz_type: 'my_viz', }; const queriesData = [ { @@ -61,15 +64,15 @@ describe('EchartsTimeseries tranformProps', () => { series: expect.arrayContaining([ expect.objectContaining({ data: [ - [new Date(599616000000), 1], - [new Date(599916000000), 3], + [599616000000, 1], + [599916000000, 3], ], name: 'San Francisco', }), expect.objectContaining({ data: [ - [new Date(599616000000), 2], - [new Date(599916000000), 4], + [599616000000, 2], + [599916000000, 4], ], name: 'New York', }), @@ -82,10 +85,11 @@ describe('EchartsTimeseries tranformProps', () => { it('should add a formula annotation to viz', () => { const formula: FormulaAnnotationLayer = { name: 'My Formula', - annotationType: 'FORMULA', + annotationType: AnnotationType.Formula, value: 'x+1', - style: 'solid', + style: AnnotationStyle.Solid, show: true, + showLabel: true, }; const chartProps = new ChartProps({ ...chartPropsConfig, @@ -105,15 +109,15 @@ describe('EchartsTimeseries tranformProps', () => { series: expect.arrayContaining([ expect.objectContaining({ data: [ - [new Date(599616000000), 1], - [new Date(599916000000), 3], + [599616000000, 1], + [599916000000, 3], ], name: 'San Francisco', }), expect.objectContaining({ data: [ - [new Date(599616000000), 2], - [new Date(599916000000), 4], + [599616000000, 2], + [599916000000, 4], ], name: 'New York', }), diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts index 96a75b7e93f72..e59cd0e35773c 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts @@ -88,6 +88,7 @@ describe('extractAnnotationLabels', () => { show: true, style: AnnotationStyle.Solid, value: 'sin(x)', + showLabel: true, }, { annotationType: AnnotationType.Formula, @@ -95,6 +96,7 @@ describe('extractAnnotationLabels', () => { show: false, style: AnnotationStyle.Solid, value: 'sin(2x)', + showLabel: true, }, { annotationType: AnnotationType.Interval, @@ -103,6 +105,7 @@ describe('extractAnnotationLabels', () => { show: true, style: AnnotationStyle.Solid, value: 1, + showLabel: true, }, { annotationType: AnnotationType.Timeseries, @@ -111,6 +114,7 @@ describe('extractAnnotationLabels', () => { style: AnnotationStyle.Dashed, sourceType: AnnotationSourceType.Line, value: 1, + showLabel: true, }, { annotationType: AnnotationType.Timeseries, @@ -119,6 +123,7 @@ describe('extractAnnotationLabels', () => { style: AnnotationStyle.Dashed, sourceType: AnnotationSourceType.Line, value: 1, + showLabel: true, }, ]; const results: AnnotationResult = { @@ -147,6 +152,7 @@ describe('evalFormula', () => { show: true, style: AnnotationStyle.Solid, value: 'x+1', + showLabel: true, }; it('Should evaluate a regular formula', () => { const data: TimeseriesDataRecord[] = [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts index 91b9872ec844a..1dbd57452fe06 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts @@ -54,18 +54,18 @@ describe('extractTimeseriesSeries', () => { id: 'Hulk', name: 'Hulk', data: [ - [new Date('2000-01-01'), null], - [new Date('2000-02-01'), 2], - [new Date('2000-03-01'), 1], + ['2000-01-01', null], + ['2000-02-01', 2], + ['2000-03-01', 1], ], }, { id: 'abc', name: 'abc', data: [ - [new Date('2000-01-01'), 2], - [new Date('2000-02-01'), 10], - [new Date('2000-03-01'), 5], + ['2000-01-01', 2], + ['2000-02-01', 10], + ['2000-03-01', 5], ], }, ]); @@ -119,16 +119,16 @@ describe('extractTimeseriesSeries', () => { id: 'abc', name: 'abc', data: [ - [new Date('2000-01-01'), null], - [new Date('2000-02-01'), 0], - [new Date('2000-03-01'), 1], - [new Date('2000-04-01'), 0], - [new Date('2000-05-01'), null], - [new Date('2000-06-01'), 0], - [new Date('2000-07-01'), 2], - [new Date('2000-08-01'), 3], - [new Date('2000-09-01'), 0], - [new Date('2000-10-01'), null], + ['2000-01-01', null], + ['2000-02-01', 0], + ['2000-03-01', 1], + ['2000-04-01', 0], + ['2000-05-01', null], + ['2000-06-01', 0], + ['2000-07-01', 2], + ['2000-08-01', 3], + ['2000-09-01', 0], + ['2000-10-01', null], ], }, ]); From 1380828bec96cb11712d382a2e76f0418f6c5b36 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 4 Jan 2022 12:30:22 +0200 Subject: [PATCH 03/20] change formula return type from Date to number --- .../plugin-chart-echarts/src/utils/annotation.ts | 4 ++-- .../test/Timeseries/transformProps.test.ts | 4 ++-- .../test/utils/annotation.test.ts | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/annotation.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/annotation.ts index 9ccd7bb2dba5a..f59dbf99fbeae 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/utils/annotation.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/annotation.ts @@ -34,11 +34,11 @@ import { export function evalFormula( formula: FormulaAnnotationLayer, data: TimeseriesDataRecord[], -): [Date, number][] { +): [number, number][] { const { value: expression } = formula; return data.map(row => [ - new Date(Number(row.__timestamp)), + Number(row.__timestamp), evalExpression(expression, row.__timestamp as number), ]); } diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts index 572675dfe798a..365d543683f93 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/Timeseries/transformProps.test.ts @@ -123,8 +123,8 @@ describe('EchartsTimeseries tranformProps', () => { }), expect.objectContaining({ data: [ - [new Date(599616000000), 599616000001], - [new Date(599916000000), 599916000001], + [599616000000, 599616000001], + [599916000000, 599916000001], ], name: 'My Formula', }), diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts index e59cd0e35773c..4d6b164924141 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/utils/annotation.test.ts @@ -18,8 +18,8 @@ */ import { AnnotationLayer, + AnnotationData, AnnotationOpacity, - AnnotationResult, AnnotationSourceType, AnnotationStyle, AnnotationType, @@ -126,7 +126,7 @@ describe('extractAnnotationLabels', () => { showLabel: true, }, ]; - const results: AnnotationResult = { + const results: AnnotationData = { 'My Interval': { columns: ['col'], records: [{ col: 1 }], @@ -161,8 +161,8 @@ describe('evalFormula', () => { ]; expect(evalFormula(layer, data)).toEqual([ - [new Date(0), 1], - [new Date(10), 11], + [0, 1], + [10, 11], ]); }); @@ -173,8 +173,8 @@ describe('evalFormula', () => { ]; expect(evalFormula({ ...layer, value: 'y = x* 2 -1' }, data)).toEqual([ - [new Date(0), -1], - [new Date(10), 19], + [0, -1], + [10, 19], ]); }); }); From 51b260bb6f97adf44be2a357030659ab4bde1f52 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 4 Jan 2022 15:35:29 +0200 Subject: [PATCH 04/20] add x_axis test coverage --- .../utils/operators/pivotOperator.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/pivotOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/pivotOperator.test.ts index 8f522cc7a2775..505404951edff 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/pivotOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/pivotOperator.test.ts @@ -105,6 +105,32 @@ test('pivot by __timestamp with groupby', () => { }); }); +test('pivot by x_axis with groupby', () => { + expect( + pivotOperator( + { + ...formData, + x_axis: 'baz', + }, + { + ...queryObject, + columns: ['foo', 'bar'], + }, + ), + ).toEqual({ + operation: 'pivot', + options: { + index: ['baz'], + columns: ['foo', 'bar'], + aggregates: { + 'count(*)': { operator: 'mean' }, + 'sum(val)': { operator: 'mean' }, + }, + drop_missing_columns: false, + }, + }); +}); + test('timecompare in formdata', () => { expect( pivotOperator( From cd9d488a9c16829caf6c2d495b732a2b2af1950e Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Mon, 10 Jan 2022 13:06:34 +0200 Subject: [PATCH 05/20] rename func and improve coverage --- .../src/MixedTimeseries/transformProps.ts | 6 +-- .../src/Timeseries/transformProps.ts | 9 ++-- .../plugin-chart-echarts/src/utils/series.ts | 9 ++-- .../test/utils/series.test.ts | 43 +++++++++++++++++-- 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/transformProps.ts index 1f39a4ade9209..886f69314e379 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/transformProps.ts @@ -40,7 +40,7 @@ import { parseYAxisBound } from '../utils/controls'; import { currentSeries, dedupSeries, - extractTimeseriesSeries, + extractSeries, getLegendProps, } from '../utils/series'; import { extractAnnotationLabels } from '../utils/annotation'; @@ -131,11 +131,11 @@ export default function transformProps( const colorScale = CategoricalColorNamespace.getScale(colorScheme as string); const rebasedDataA = rebaseTimeseriesDatum(data1, verboseMap); - const rawSeriesA = extractTimeseriesSeries(rebasedDataA, { + const rawSeriesA = extractSeries(rebasedDataA, { fillNeighborValue: stack ? 0 : undefined, }); const rebasedDataB = rebaseTimeseriesDatum(data2, verboseMap); - const rawSeriesB = extractTimeseriesSeries(rebasedDataB, { + const rawSeriesB = extractSeries(rebasedDataB, { fillNeighborValue: stackB ? 0 : undefined, }); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts index f6190904a489d..4e32f5c7aedd2 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts @@ -43,7 +43,7 @@ import { parseYAxisBound } from '../utils/controls'; import { currentSeries, dedupSeries, - extractTimeseriesSeries, + extractSeries, getColtypesMapping, getLegendProps, } from '../utils/series'; @@ -127,13 +127,10 @@ export default function transformProps( const colorScale = CategoricalColorNamespace.getScale(colorScheme as string); const rebasedData = rebaseTimeseriesDatum(data, verboseMap); const xAxisCol = xAxisOrig || DTTM_ALIAS; - const rawSeries = extractTimeseriesSeries(rebasedData, { + const rawSeries = extractSeries(rebasedData, { fillNeighborValue: stack && !forecastEnabled ? 0 : undefined, xAxis: xAxisCol, - removeNulls: [ - EchartsTimeseriesSeriesType.Line, - EchartsTimeseriesSeriesType.Scatter, - ].includes(seriesType), + removeNulls: seriesType === EchartsTimeseriesSeriesType.Scatter, }); const seriesContexts = extractForecastSeriesContexts( Object.values(rawSeries).map(series => series.name as string), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts index 763713f520733..d4c53d9df0f89 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/series.ts @@ -26,7 +26,6 @@ import { GenericDataType, NumberFormatter, TimeFormatter, - TimeseriesDataRecord, } from '@superset-ui/core'; import { format, LegendComponentOption, SeriesOption } from 'echarts'; import { NULL_STRING, TIMESERIES_CONSTANTS } from '../constants'; @@ -37,8 +36,8 @@ function isDefined(value: T | undefined | null): boolean { return value !== undefined && value !== null; } -export function extractTimeseriesSeries( - data: TimeseriesDataRecord[], +export function extractSeries( + data: DataRecord[], opts: { fillNeighborValue?: number; xAxis?: string; @@ -47,7 +46,7 @@ export function extractTimeseriesSeries( ): SeriesOption[] { const { fillNeighborValue, xAxis = DTTM_ALIAS, removeNulls = false } = opts; if (data.length === 0) return []; - const rows: TimeseriesDataRecord[] = data.map(datum => ({ + const rows: DataRecord[] = data.map(datum => ({ ...datum, [xAxis]: datum[xAxis], })); @@ -70,7 +69,7 @@ export function extractTimeseriesSeries( : row[key], ]; }) - .filter(row => !removeNulls || row[1] !== null), + .filter(obs => !removeNulls || (obs[0] !== null && obs[1] !== null)), })); } diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts index 1dbd57452fe06..36cb2047f3654 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/utils/series.test.ts @@ -20,7 +20,7 @@ import { getNumberFormatter, getTimeFormatter } from '@superset-ui/core'; import { dedupSeries, extractGroupbyLabel, - extractTimeseriesSeries, + extractSeries, formatSeriesName, getChartPadding, getLegendProps, @@ -30,7 +30,7 @@ import { LegendOrientation, LegendType } from '../../src/types'; import { defaultLegendPadding } from '../../src/defaults'; import { NULL_STRING } from '../../src/constants'; -describe('extractTimeseriesSeries', () => { +describe('extractSeries', () => { it('should generate a valid ECharts timeseries series object', () => { const data = [ { @@ -49,7 +49,7 @@ describe('extractTimeseriesSeries', () => { abc: 5, }, ]; - expect(extractTimeseriesSeries(data)).toEqual([ + expect(extractSeries(data)).toEqual([ { id: 'Hulk', name: 'Hulk', @@ -71,6 +71,41 @@ describe('extractTimeseriesSeries', () => { ]); }); + it('should remove rows that have a null x-value', () => { + const data = [ + { + x: 1, + Hulk: null, + abc: 2, + }, + { + x: null, + Hulk: 2, + abc: 10, + }, + { + x: 2, + Hulk: 1, + abc: 5, + }, + ]; + expect(extractSeries(data, { xAxis: 'x', removeNulls: true })).toEqual([ + { + id: 'Hulk', + name: 'Hulk', + data: [[2, 1]], + }, + { + id: 'abc', + name: 'abc', + data: [ + [1, 2], + [2, 5], + ], + }, + ]); + }); + it('should do missing value imputation', () => { const data = [ { @@ -114,7 +149,7 @@ describe('extractTimeseriesSeries', () => { abc: null, }, ]; - expect(extractTimeseriesSeries(data, { fillNeighborValue: 0 })).toEqual([ + expect(extractSeries(data, { fillNeighborValue: 0 })).toEqual([ { id: 'abc', name: 'abc', From dbc262940f91601c0e555075d85b82e61a619bb9 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Mon, 10 Jan 2022 16:00:26 +0200 Subject: [PATCH 06/20] add x-axis control to bar chart --- .../plugin-chart-echarts/src/Timeseries/Area/index.ts | 6 +++++- .../src/Timeseries/Regular/Bar/controlPanel.tsx | 4 +++- .../src/Timeseries/Regular/Scatter/controlPanel.tsx | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts index 9e664c280e8b3..49adccb5ac89f 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts @@ -22,6 +22,8 @@ import { ChartPlugin, AnnotationType, Behavior, + isFeatureEnabled, + FeatureFlag, } from '@superset-ui/core'; import buildQuery from '../buildQuery'; import controlPanel from './controlPanel'; @@ -62,7 +64,9 @@ export default class EchartsAreaChartPlugin extends ChartPlugin< AnnotationType.Interval, AnnotationType.Timeseries, ], - name: t('Time-series Area Chart'), + name: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t('Area Chart v2') + : t('Time-series Area Chart'), tags: [ t('ECharts'), t('Predictive'), diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/controlPanel.tsx index 99023364249d2..96cb0b6bda60d 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Bar/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -35,6 +35,7 @@ import { legendSection, richTooltipSection, showValueSection, + xAxisControl, } from '../../../controls'; const { @@ -56,6 +57,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], [ diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/controlPanel.tsx index a2a34465679ac..21110c82cfefa 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Regular/Scatter/controlPanel.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { FeatureFlag, isFeatureEnabled, t } from '@superset-ui/core'; import { ControlPanelConfig, ControlPanelsContainerProps, @@ -32,6 +32,7 @@ import { legendSection, richTooltipSection, showValueSection, + xAxisControl, } from '../../../controls'; const { @@ -52,6 +53,7 @@ const config: ControlPanelConfig = { label: t('Query'), expanded: true, controlSetRows: [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? [xAxisControl] : [], ['metrics'], ['groupby'], ['adhoc_filters'], From a3ec47674239b45441b5a7b8eef5d2aec375134a Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 11 Jan 2022 12:00:37 +0200 Subject: [PATCH 07/20] remove redundant console.log --- .../plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx index e68fa88693aaf..2bf103e2bd9e4 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/EchartsTimeseries.tsx @@ -130,9 +130,7 @@ export default function EchartsTimeseries({ } // Ensure that double-click events do not trigger single click event. So we put it in the timer. clickTimer.current = setTimeout(() => { - const { seriesName: name, value } = props; - const xValue = value[0].getTime?.() || value[0]; - console.log(xValue); + const { seriesName: name } = props; const values = Object.values(selectedValues); if (values.includes(name)) { handleChange(values.filter(v => v !== name)); From 20c9e4366cf7a06d94d0ec838712816a7ee305f6 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 11 Jan 2022 14:16:47 +0200 Subject: [PATCH 08/20] fix description --- .../plugin-chart-echarts/src/Timeseries/Area/index.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts index 49adccb5ac89f..0ec87e64939f5 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts @@ -54,9 +54,13 @@ export default class EchartsAreaChartPlugin extends ChartPlugin< behaviors: [Behavior.INTERACTIVE_CHART], category: t('Evolution'), credits: ['https://echarts.apache.org'], - description: t( - 'Area charts are similar to line charts in that they represent variables with the same scale, but area charts stack the metrics on top of each other.', - ), + description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? t( + 'Time-series Area chart are similar to line chart in that they represent variables with the same scale, but area charts stack the metrics on top of each other. An area chart in Superset can be stream, stack, or expand.', + ) + : t( + 'Area charts are similar to line charts in that they represent variables with the same scale, but area charts stack the metrics on top of each other.', + ), exampleGallery: [{ url: example1 }], supportedAnnotationTypes: [ AnnotationType.Event, From 6ef3f255943a6589c919047c71bbb44094260ee9 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 11 Jan 2022 14:58:02 +0200 Subject: [PATCH 09/20] make x-axis control mandatory --- .../plugins/plugin-chart-echarts/src/controls.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx index 978367b54a657..f4c3012011a8e 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/controls.tsx @@ -17,7 +17,7 @@ * under the License. */ import React from 'react'; -import { t } from '@superset-ui/core'; +import { t, validateNonEmpty } from '@superset-ui/core'; import { ControlPanelsContainerProps, ControlSetRow, @@ -143,9 +143,8 @@ export const xAxisControl = { label: t('X-axis'), default: null, multi: false, - description: t( - 'Dimension to use on x-axis. Leave blank to default to temporal column.', - ), + description: t('Dimension to use on x-axis.'), + validators: [validateNonEmpty], }, }; From bbbe3cca6a216e390c814c726566bd2485d29fa8 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 11 Jan 2022 15:09:26 +0200 Subject: [PATCH 10/20] =?UTF-8?q?=F0=9F=99=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts index 0ec87e64939f5..ee7c89993cb7e 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/Area/index.ts @@ -56,10 +56,10 @@ export default class EchartsAreaChartPlugin extends ChartPlugin< credits: ['https://echarts.apache.org'], description: isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? t( - 'Time-series Area chart are similar to line chart in that they represent variables with the same scale, but area charts stack the metrics on top of each other. An area chart in Superset can be stream, stack, or expand.', + 'Area charts are similar to line charts in that they represent variables with the same scale, but area charts stack the metrics on top of each other.', ) : t( - 'Area charts are similar to line charts in that they represent variables with the same scale, but area charts stack the metrics on top of each other.', + 'Time-series Area chart are similar to line chart in that they represent variables with the same scale, but area charts stack the metrics on top of each other. An area chart in Superset can be stream, stack, or expand.', ), exampleGallery: [{ url: example1 }], supportedAnnotationTypes: [ From 7891cced520c7a5b56321a81da9f5d6372d3f27d Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 12 Jan 2022 11:27:25 +0200 Subject: [PATCH 11/20] fix x-axis formatter --- .../plugin-chart-echarts/src/Timeseries/transformProps.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts index 4e32f5c7aedd2..43fd290c7c777 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts @@ -250,7 +250,10 @@ export default function transformProps( xAxisDataType === GenericDataType.TEMPORAL ? getTooltipTimeFormatter(tooltipTimeFormat) : String; - const xAxisFormatter = getXAxisFormatter(xAxisTimeFormat); + const xAxisFormatter = + xAxisDataType === GenericDataType.TEMPORAL + ? getXAxisFormatter(xAxisTimeFormat) + : String; const labelMap = series.reduce( (acc: Record, datum) => { From 57d0c7ffb3dc9cfc1cf390256721a674b1cbfe6d Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 12 Jan 2022 11:35:58 +0200 Subject: [PATCH 12/20] fix showValues --- .../plugin-chart-echarts/src/Timeseries/transformProps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts index 43fd290c7c777..df56fade3b049 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/transformProps.ts @@ -156,7 +156,7 @@ export default function transformProps( rebasedData.forEach(data => { const values = Object.keys(data).reduce((prev, curr) => { - if (curr === DTTM_ALIAS) { + if (curr === xAxisCol) { return prev; } const value = data[curr] || 0; From cf49e33630d82f9e3dfa7a7d1d088fe015424a9c Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 13 Jan 2022 10:56:23 +0200 Subject: [PATCH 13/20] fix implicit rDTTM_ALIAS references in postProcessing --- .../src/operators/prophetOperator.ts | 2 ++ .../src/operators/rollingWindowOperator.ts | 8 ++--- .../src/operators/sortOperator.ts | 6 ++-- .../utils/operators/prophetOperator.test.ts | 33 ++++++++++++++++++- .../test/utils/operators/sortOperator.test.ts | 16 +++++++++ superset/utils/pandas_postprocessing.py | 11 ++++--- 6 files changed, 65 insertions(+), 11 deletions(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts index 1b208034db852..c48c8c9f14586 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts @@ -18,6 +18,7 @@ */ import { PostProcessingProphet } from '@superset-ui/core'; import { PostProcessingFactory } from './types'; +import { TIME_COLUMN } from './utils'; export const prophetOperator: PostProcessingFactory< PostProcessingProphet | undefined @@ -32,6 +33,7 @@ export const prophetOperator: PostProcessingFactory< yearly_seasonality: formData.forecastSeasonalityYearly, weekly_seasonality: formData.forecastSeasonalityWeekly, daily_seasonality: formData.forecastSeasonalityDaily, + index: formData.x_axis || TIME_COLUMN, }, }; } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts index f8dd40137bfcb..d4c04ec789668 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/rollingWindowOperator.ts @@ -18,12 +18,12 @@ * under the License. */ import { - ensureIsInt, + ComparisionType, ensureIsArray, - RollingType, - PostProcessingRolling, + ensureIsInt, PostProcessingCum, - ComparisionType, + PostProcessingRolling, + RollingType, } from '@superset-ui/core'; import { getMetricOffsetsMap, diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts index 9a86f4b371e65..c705f2d8922d6 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts @@ -24,15 +24,17 @@ import { TIME_COLUMN } from './utils'; export const sortOperator: PostProcessingFactory< PostProcessingSort | undefined > = (formData, queryObject) => { + const { x_axis: xAxis } = formData; if ( - queryObject.is_timeseries && + (xAxis || queryObject.is_timeseries) && Object.values(RollingType).includes(formData.rolling_type) ) { + const index = xAxis || TIME_COLUMN; return { operation: 'sort', options: { columns: { - [TIME_COLUMN]: true, + [index]: true, }, }, }; diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts index d79868ad08c2e..8f1135b68a7e9 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts @@ -18,6 +18,7 @@ */ import { QueryObject, SqlaFormData } from '@superset-ui/core'; import { prophetOperator } from '../../../src'; +import { TIME_COLUMN } from '../../../lib'; const formData: SqlaFormData = { metrics: [ @@ -42,7 +43,7 @@ test('should skip prophetOperator', () => { expect(prophetOperator(formData, queryObject)).toEqual(undefined); }); -test('should do prophetOperator', () => { +test('should do prophetOperator with default index', () => { expect( prophetOperator( { @@ -65,6 +66,36 @@ test('should do prophetOperator', () => { yearly_seasonality: true, weekly_seasonality: false, daily_seasonality: false, + index: TIME_COLUMN, + }, + }); +}); + +test('should do prophetOperator over named column', () => { + expect( + prophetOperator( + { + ...formData, + x_axis: 'ds', + forecastEnabled: true, + forecastPeriods: '3', + forecastInterval: '5', + forecastSeasonalityYearly: true, + forecastSeasonalityWeekly: false, + forecastSeasonalityDaily: false, + }, + queryObject, + ), + ).toEqual({ + operation: 'prophet', + options: { + time_grain: 'P1Y', + periods: 3.0, + confidence_interval: 5.0, + yearly_seasonality: true, + weekly_seasonality: false, + daily_seasonality: false, + index: 'ds', }, }); }); diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/sortOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/sortOperator.test.ts index e8dfe3d23df44..54cf348348895 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/sortOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/sortOperator.test.ts @@ -125,3 +125,19 @@ test('sort by __timestamp', () => { }, }); }); + +test('sort by named x-axis', () => { + expect( + sortOperator( + { ...formData, x_axis: 'ds', rolling_type: 'cumsum' }, + { ...queryObject }, + ), + ).toEqual({ + operation: 'sort', + options: { + columns: { + ds: true, + }, + }, + }); +}); diff --git a/superset/utils/pandas_postprocessing.py b/superset/utils/pandas_postprocessing.py index ea4e40986bf81..8906e2c9038e8 100644 --- a/superset/utils/pandas_postprocessing.py +++ b/superset/utils/pandas_postprocessing.py @@ -785,6 +785,7 @@ def prophet( # pylint: disable=too-many-arguments yearly_seasonality: Optional[Union[bool, int]] = None, weekly_seasonality: Optional[Union[bool, int]] = None, daily_seasonality: Optional[Union[bool, int]] = None, + index: Optional[str] = None, ) -> DataFrame: """ Add forecasts to each series in a timeseries dataframe, along with confidence @@ -808,8 +809,10 @@ def prophet( # pylint: disable=too-many-arguments :param daily_seasonality: Should daily seasonality be applied. An integer value will specify Fourier order of seasonality, `None` will automatically detect seasonality. + :param index: the name of the column containing the x-axis data :return: DataFrame with contributions, with temporal column at beginning if present """ + index = index or DTTM_ALIAS # validate inputs if not time_grain: raise QueryObjectValidationError(_("Time grain missing")) @@ -826,15 +829,15 @@ def prophet( # pylint: disable=too-many-arguments raise QueryObjectValidationError( _("Confidence interval must be between 0 and 1 (exclusive)") ) - if DTTM_ALIAS not in df.columns: + if index not in df.columns: raise QueryObjectValidationError(_("DataFrame must include temporal column")) if len(df.columns) < 2: raise QueryObjectValidationError(_("DataFrame include at least one series")) target_df = DataFrame() - for column in [column for column in df.columns if column != DTTM_ALIAS]: + for column in [column for column in df.columns if column != index]: fit_df = _prophet_fit_and_predict( - df=df[[DTTM_ALIAS, column]].rename(columns={DTTM_ALIAS: "ds", column: "y"}), + df=df[[index, column]].rename(columns={index: "ds", column: "y"}), confidence_interval=confidence_interval, yearly_seasonality=_prophet_parse_seasonality(yearly_seasonality), weekly_seasonality=_prophet_parse_seasonality(weekly_seasonality), @@ -855,7 +858,7 @@ def prophet( # pylint: disable=too-many-arguments for new_column in new_columns: target_df = target_df.assign(**{new_column: fit_df[new_column]}) target_df.reset_index(level=0, inplace=True) - return target_df.rename(columns={"ds": DTTM_ALIAS}) + return target_df.rename(columns={"ds": index}) def boxplot( From 8b7a6381e386de08420a5fe2b1c5f9becb986692 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 13 Jan 2022 11:31:29 +0200 Subject: [PATCH 14/20] replace TIME_COLUMN with DTTM_ALIAS --- .../src/operators/pivotOperator.ts | 5 +++-- .../src/operators/prophetOperator.ts | 5 ++--- .../src/operators/resampleOperator.ts | 5 ++--- .../src/operators/sortOperator.ts | 5 ++--- .../src/operators/utils/constants.ts | 1 - .../src/operators/utils/index.ts | 2 +- .../test/utils/operators/prophetOperator.test.ts | 5 ++--- .../src/BigNumber/BigNumberWithTrendline/buildQuery.ts | 7 ++----- .../src/BigNumber/BigNumberWithTrendline/transformProps.ts | 6 +++--- 9 files changed, 17 insertions(+), 24 deletions(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts index 629fc18cacf90..49467f3b5f891 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts @@ -17,13 +17,14 @@ * under the License. */ import { + DTTM_ALIAS, ensureIsArray, getColumnLabel, getMetricLabel, PostProcessingPivot, } from '@superset-ui/core'; import { PostProcessingFactory } from './types'; -import { TIME_COLUMN, isValidTimeCompare } from './utils'; +import { isValidTimeCompare } from './utils'; import { timeComparePivotOperator } from './timeComparePivotOperator'; export const pivotOperator: PostProcessingFactory< @@ -32,7 +33,7 @@ export const pivotOperator: PostProcessingFactory< const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel); const { x_axis: xAxis } = formData; if ((xAxis || queryObject.is_timeseries) && metricLabels.length) { - const index = !xAxis || queryObject.is_timeseries ? TIME_COLUMN : xAxis; + const index = !xAxis || queryObject.is_timeseries ? DTTM_ALIAS : xAxis; if (isValidTimeCompare(formData, queryObject)) { return timeComparePivotOperator(formData, queryObject); } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts index c48c8c9f14586..640cb8bda23d3 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/prophetOperator.ts @@ -16,9 +16,8 @@ * specific language governing permissions and limitationsxw * under the License. */ -import { PostProcessingProphet } from '@superset-ui/core'; +import { DTTM_ALIAS, PostProcessingProphet } from '@superset-ui/core'; import { PostProcessingFactory } from './types'; -import { TIME_COLUMN } from './utils'; export const prophetOperator: PostProcessingFactory< PostProcessingProphet | undefined @@ -33,7 +32,7 @@ export const prophetOperator: PostProcessingFactory< yearly_seasonality: formData.forecastSeasonalityYearly, weekly_seasonality: formData.forecastSeasonalityWeekly, daily_seasonality: formData.forecastSeasonalityDaily, - index: formData.x_axis || TIME_COLUMN, + index: formData.x_axis || DTTM_ALIAS, }, }; } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts index a43838d710b8c..f9f9c8c2b3710 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts @@ -17,9 +17,8 @@ * specific language governing permissions and limitationsxw * under the License. */ -import { PostProcessingResample } from '@superset-ui/core'; +import { DTTM_ALIAS, PostProcessingResample } from '@superset-ui/core'; import { PostProcessingFactory } from './types'; -import { TIME_COLUMN } from './utils'; export const resampleOperator: PostProcessingFactory< PostProcessingResample | undefined @@ -34,7 +33,7 @@ export const resampleOperator: PostProcessingFactory< method: resampleMethod, rule: resampleRule, fill_value: resampleZeroFill ? 0 : null, - time_column: TIME_COLUMN, + time_column: DTTM_ALIAS, }, }; } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts index c705f2d8922d6..9443bb729bbb9 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/sortOperator.ts @@ -17,9 +17,8 @@ * specific language governing permissions and limitationsxw * under the License. */ -import { PostProcessingSort, RollingType } from '@superset-ui/core'; +import { DTTM_ALIAS, PostProcessingSort, RollingType } from '@superset-ui/core'; import { PostProcessingFactory } from './types'; -import { TIME_COLUMN } from './utils'; export const sortOperator: PostProcessingFactory< PostProcessingSort | undefined @@ -29,7 +28,7 @@ export const sortOperator: PostProcessingFactory< (xAxis || queryObject.is_timeseries) && Object.values(RollingType).includes(formData.rolling_type) ) { - const index = xAxis || TIME_COLUMN; + const index = xAxis || DTTM_ALIAS; return { operation: 'sort', options: { diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/constants.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/constants.ts index cb01a2e6c5cb7..6f2f59b8d801c 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/constants.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/constants.ts @@ -18,4 +18,3 @@ * under the License. */ export const TIME_COMPARISON_SEPARATOR = '__'; -export const TIME_COLUMN = '__timestamp'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/index.ts index acf072f74678b..d591dbd23edde 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/utils/index.ts @@ -19,4 +19,4 @@ */ export { getMetricOffsetsMap } from './getMetricOffsetsMap'; export { isValidTimeCompare } from './isValidTimeCompare'; -export { TIME_COMPARISON_SEPARATOR, TIME_COLUMN } from './constants'; +export { TIME_COMPARISON_SEPARATOR } from './constants'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts index 8f1135b68a7e9..78d4bc9765b6e 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/prophetOperator.test.ts @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -import { QueryObject, SqlaFormData } from '@superset-ui/core'; +import { DTTM_ALIAS, QueryObject, SqlaFormData } from '@superset-ui/core'; import { prophetOperator } from '../../../src'; -import { TIME_COLUMN } from '../../../lib'; const formData: SqlaFormData = { metrics: [ @@ -66,7 +65,7 @@ test('should do prophetOperator with default index', () => { yearly_seasonality: true, weekly_seasonality: false, daily_seasonality: false, - index: TIME_COLUMN, + index: DTTM_ALIAS, }, }); }); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts index 1a9096ff77906..b4dcd2a092f65 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts @@ -22,10 +22,7 @@ import { PostProcessingResample, QueryFormData, } from '@superset-ui/core'; -import { - rollingWindowOperator, - TIME_COLUMN, -} from '@superset-ui/chart-controls'; +import { rollingWindowOperator } from '@superset-ui/chart-controls'; const TIME_GRAIN_MAP: Record = { PT1S: 'S', @@ -65,7 +62,7 @@ export default function buildQuery(formData: QueryFormData) { method: 'asfreq', rule, fill_value: null, - time_column: TIME_COLUMN, + time_column: DTTM_ALIAS, }, }; } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts index 5054d99515ccf..892a81e5c11a5 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts @@ -17,6 +17,7 @@ * under the License. */ import { + DTTM_ALIAS, extractTimegrain, getNumberFormatter, NumberFormats, @@ -54,7 +55,6 @@ export function renderTooltipFactory( }; } -const TIME_COLUMN = '__timestamp'; const formatPercentChange = getNumberFormatter( NumberFormats.PERCENT_SIGNED_1_POINT, ); @@ -97,7 +97,7 @@ export default function transformProps( let trendLineData; let percentChange = 0; let bigNumber = data.length === 0 ? null : data[0][metricName]; - let timestamp = data.length === 0 ? null : data[0][TIME_COLUMN]; + let timestamp = data.length === 0 ? null : data[0][DTTM_ALIAS]; let bigNumberFallback; const metricColtypeIndex = colnames.findIndex(name => name === metricName); @@ -106,7 +106,7 @@ export default function transformProps( if (data.length > 0) { const sortedData = (data as BigNumberDatum[]) - .map(d => [d[TIME_COLUMN], parseMetricValue(d[metricName])]) + .map(d => [d[DTTM_ALIAS], parseMetricValue(d[metricName])]) // sort in time descending order .sort((a, b) => (a[0] !== null && b[0] !== null ? b[0] - a[0] : 0)); From 934165d435973d8725f06a3a6320f63ae2569578 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 13 Jan 2022 13:42:55 +0200 Subject: [PATCH 15/20] fix remaining implicit indexes --- .../src/operators/pivotOperator.ts | 3 +-- .../src/operators/resampleOperator.ts | 2 +- .../src/operators/timeComparePivotOperator.ts | 7 ++--- .../utils/operators/resampleOperator.test.ts | 24 ++++++++++++++++- .../operators/timeCompareOperator.test.ts | 26 +++++++++++++++++++ superset/common/query_context_processor.py | 12 ++++++--- 6 files changed, 63 insertions(+), 11 deletions(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts index 49467f3b5f891..e1e1fde95668d 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts @@ -33,7 +33,6 @@ export const pivotOperator: PostProcessingFactory< const metricLabels = ensureIsArray(queryObject.metrics).map(getMetricLabel); const { x_axis: xAxis } = formData; if ((xAxis || queryObject.is_timeseries) && metricLabels.length) { - const index = !xAxis || queryObject.is_timeseries ? DTTM_ALIAS : xAxis; if (isValidTimeCompare(formData, queryObject)) { return timeComparePivotOperator(formData, queryObject); } @@ -41,7 +40,7 @@ export const pivotOperator: PostProcessingFactory< return { operation: 'pivot', options: { - index: [index], + index: [xAxis || DTTM_ALIAS], columns: ensureIsArray(queryObject.columns).map(getColumnLabel), // Create 'dummy' mean aggregates to assign cell values in pivot table // use the 'mean' aggregates to avoid drop NaN. PR: https://github.com/apache-superset/superset-ui/pull/1231 diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts index f9f9c8c2b3710..7fb18f5cc8b3f 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/resampleOperator.ts @@ -33,7 +33,7 @@ export const resampleOperator: PostProcessingFactory< method: resampleMethod, rule: resampleRule, fill_value: resampleZeroFill ? 0 : null, - time_column: DTTM_ALIAS, + time_column: formData.x_axis || DTTM_ALIAS, }, }; } diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts b/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts index 4b5458e67d841..9e16d294004f0 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts @@ -19,10 +19,11 @@ */ import { ComparisionType, - PostProcessingPivot, - NumpyFunction, + DTTM_ALIAS, ensureIsArray, getColumnLabel, + NumpyFunction, + PostProcessingPivot, } from '@superset-ui/core'; import { getMetricOffsetsMap, @@ -57,7 +58,7 @@ export const timeComparePivotOperator: PostProcessingFactory< return { operation: 'pivot', options: { - index: ['__timestamp'], + index: [formData.x_axis || DTTM_ALIAS], columns: ensureIsArray(queryObject.columns).map(getColumnLabel), aggregates: comparisonType === ComparisionType.Values ? valuesAgg : changeAgg, diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts index bce5a84abc556..cbb1e232d06d5 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts @@ -62,7 +62,7 @@ test('should skip resampleOperator', () => { ).toEqual(undefined); }); -test('should do resample', () => { +test('should do resample on implicit time column', () => { expect( resampleOperator( { ...formData, resample_method: 'ffill', resample_rule: '1D' }, @@ -79,6 +79,28 @@ test('should do resample', () => { }); }); +test('should do resample on x-axis', () => { + expect( + resampleOperator( + { + ...formData, + x_axis: 'ds', + resample_method: 'ffill', + resample_rule: '1D', + }, + queryObject, + ), + ).toEqual({ + operation: 'resample', + options: { + method: 'ffill', + rule: '1D', + fill_value: null, + time_column: 'ds', + }, + }); +}); + test('should do zerofill resample', () => { expect( resampleOperator( diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/timeCompareOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/timeCompareOperator.test.ts index 0b51fa7877d75..c2fcb75f84156 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/timeCompareOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/timeCompareOperator.test.ts @@ -174,3 +174,29 @@ test('time compare pivot: difference/percentage/ratio', () => { }); }); }); + +test('time compare pivot on x-axis', () => { + expect( + timeComparePivotOperator( + { + ...formData, + comparison_type: 'values', + time_compare: ['1 year ago', '1 year later'], + x_axis: 'ds', + }, + queryObject, + ), + ).toEqual({ + operation: 'pivot', + options: { + aggregates: { + 'count(*)': { operator: 'mean' }, + 'count(*)__1 year ago': { operator: 'mean' }, + 'count(*)__1 year later': { operator: 'mean' }, + }, + drop_missing_columns: false, + columns: [], + index: ['ds'], + }, + }); +}); diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index 9c641cb3ea7c0..a313d9d8258d6 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -306,10 +306,14 @@ def processing_time_offsets( # pylint: disable=too-many-locals # 2. rename extra query columns offset_metrics_df = offset_metrics_df.rename(columns=metrics_mapping) - # 3. set time offset for dttm column - offset_metrics_df[DTTM_ALIAS] = offset_metrics_df[ - DTTM_ALIAS - ] - DateOffset(**normalize_time_delta(offset)) + # 3. set time offset for index + # TODO: add x-axis to QueryObject, potentially as an array for + # multi-dimensional charts + granularity = query_object.granularity + index = granularity if granularity in df.columns else DTTM_ALIAS + offset_metrics_df[index] = offset_metrics_df[index] - DateOffset( + **normalize_time_delta(offset) + ) # df left join `offset_metrics_df` offset_df = df_utils.left_join_df( From 1e1f40543ea3479d044a15e73a6b3ad21df93abc Mon Sep 17 00:00:00 2001 From: Erik Ritter Date: Thu, 13 Jan 2022 10:11:18 -0800 Subject: [PATCH 16/20] fix: Disable filtering on wide result sets (#18021) --- .../src/SqlLab/components/ResultSet/index.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/superset-frontend/src/SqlLab/components/ResultSet/index.tsx b/superset-frontend/src/SqlLab/components/ResultSet/index.tsx index fde7f19c1a460..a341e82e9f33a 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet/index.tsx +++ b/superset-frontend/src/SqlLab/components/ResultSet/index.tsx @@ -37,7 +37,9 @@ import { SaveDatasetModal } from 'src/SqlLab/components/SaveDatasetModal'; import { UserWithPermissionsAndRoles } from 'src/types/bootstrapTypes'; import ProgressBar from 'src/components/ProgressBar'; import Loading from 'src/components/Loading'; -import FilterableTable from 'src/components/FilterableTable/FilterableTable'; +import FilterableTable, { + MAX_COLUMNS_FOR_TABLE, +} from 'src/components/FilterableTable/FilterableTable'; import CopyToClipboard from 'src/components/CopyToClipboard'; import { prepareCopyToClipboardTabularData } from 'src/utils/common'; import { exploreChart } from 'src/explore/exploreUtils'; @@ -560,7 +562,12 @@ export default class ResultSet extends React.PureComponent< onChange={this.changeSearch} value={this.state.searchText} className="form-control input-sm" - placeholder={t('Filter results')} + disabled={columns.length > MAX_COLUMNS_FOR_TABLE} + placeholder={ + columns.length > MAX_COLUMNS_FOR_TABLE + ? t('Too many columns to filter') + : t('Filter results') + } /> )} From 4a6f08eca517bf88bbe00f090bc573b2adca4153 Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Thu, 13 Jan 2022 21:47:07 -0800 Subject: [PATCH 17/20] fix: handle null values in time-series table (#18039) --- .../src/visualizations/TimeTable/TimeTable.jsx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/superset-frontend/src/visualizations/TimeTable/TimeTable.jsx b/superset-frontend/src/visualizations/TimeTable/TimeTable.jsx index b02eb985a0abc..adef1833f3298 100644 --- a/superset-frontend/src/visualizations/TimeTable/TimeTable.jsx +++ b/superset-frontend/src/visualizations/TimeTable/TimeTable.jsx @@ -192,14 +192,17 @@ const TimeTable = ({ } else { v = reversedEntries[timeLag][valueField]; } - if (column.comparisonType === 'diff') { - v = recent - v; - } else if (column.comparisonType === 'perc') { - v = recent / v; - } else if (column.comparisonType === 'perc_change') { - v = recent / v - 1; + if (typeof v === 'number' || typeof recent === 'number') { + if (column.comparisonType === 'diff') { + v = recent - v; + } else if (column.comparisonType === 'perc') { + v = recent / v; + } else if (column.comparisonType === 'perc_change') { + v = recent / v - 1; + } + } else { + v = 'N/A'; } - v = v || 0; } else if (column.colType === 'contrib') { // contribution to column total v = From 134f5714762f43e3c7484d13fd087cb2ff14b81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BA=B7ng=20Minh=20D=C5=A9ng?= Date: Fri, 14 Jan 2022 16:07:17 +0700 Subject: [PATCH 18/20] cleanup column_type_mappings (#17569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Đặng Minh Dũng --- superset/db_engine_specs/base.py | 137 ++++++++++++--------------- superset/db_engine_specs/mysql.py | 29 ++---- superset/db_engine_specs/postgres.py | 38 +++----- superset/db_engine_specs/presto.py | 71 +++++--------- 4 files changed, 105 insertions(+), 170 deletions(-) diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index cd416dc12a3a1..f579fc2502768 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -53,7 +53,7 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import quoted_name, text from sqlalchemy.sql.expression import ColumnClause, Select, TextAsFrom, TextClause -from sqlalchemy.types import String, TypeEngine, UnicodeText +from sqlalchemy.types import TypeEngine from typing_extensions import TypedDict from superset import security_manager, sql_parse @@ -71,6 +71,12 @@ from superset.connectors.sqla.models import TableColumn from superset.models.core import Database +ColumnTypeMapping = Tuple[ + Pattern[str], + Union[TypeEngine, Callable[[Match[str]], TypeEngine]], + GenericDataType, +] + logger = logging.getLogger() @@ -156,26 +162,37 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods engine = "base" # str as defined in sqlalchemy.engine.engine engine_aliases: Set[str] = set() - engine_name: Optional[ - str - ] = None # used for user messages, overridden in child classes + engine_name: Optional[str] = None # for user messages, overridden in child classes _date_trunc_functions: Dict[str, str] = {} _time_grain_expressions: Dict[Optional[str], str] = {} - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = ( + column_type_mappings: Tuple[ColumnTypeMapping, ...] = ( + ( + re.compile(r"^string", re.IGNORECASE), + types.String(), + GenericDataType.STRING, + ), + ( + re.compile(r"^n((var)?char|text)", re.IGNORECASE), + types.UnicodeText(), + GenericDataType.STRING, + ), + ( + re.compile(r"^(var)?char", re.IGNORECASE), + types.String(), + GenericDataType.STRING, + ), + ( + re.compile(r"^(tiny|medium|long)?text", re.IGNORECASE), + types.String(), + GenericDataType.STRING, + ), ( re.compile(r"^smallint", re.IGNORECASE), types.SmallInteger(), GenericDataType.NUMERIC, ), ( - re.compile(r"^int.*", re.IGNORECASE), + re.compile(r"^int(eger)?", re.IGNORECASE), types.Integer(), GenericDataType.NUMERIC, ), @@ -184,6 +201,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods types.BigInteger(), GenericDataType.NUMERIC, ), + (re.compile(r"^long", re.IGNORECASE), types.Float(), GenericDataType.NUMERIC,), ( re.compile(r"^decimal", re.IGNORECASE), types.Numeric(), @@ -222,26 +240,10 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods GenericDataType.NUMERIC, ), ( - re.compile(r"^string", re.IGNORECASE), - types.String(), - utils.GenericDataType.STRING, - ), - ( - re.compile(r"^N((VAR)?CHAR|TEXT)", re.IGNORECASE), - UnicodeText(), - utils.GenericDataType.STRING, - ), - ( - re.compile(r"^((VAR)?CHAR|TEXT|STRING)", re.IGNORECASE), - String(), - utils.GenericDataType.STRING, - ), - ( - re.compile(r"^((TINY|MEDIUM|LONG)?TEXT)", re.IGNORECASE), - String(), - utils.GenericDataType.STRING, + re.compile(r"^timestamp", re.IGNORECASE), + types.TIMESTAMP(), + GenericDataType.TEMPORAL, ), - (re.compile(r"^LONG", re.IGNORECASE), types.Float(), GenericDataType.NUMERIC,), ( re.compile(r"^datetime", re.IGNORECASE), types.DateTime(), @@ -252,19 +254,14 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods types.DateTime(), GenericDataType.TEMPORAL, ), - ( - re.compile(r"^timestamp", re.IGNORECASE), - types.TIMESTAMP(), - GenericDataType.TEMPORAL, - ), + (re.compile(r"^time", re.IGNORECASE), types.Time(), GenericDataType.TEMPORAL,), ( re.compile(r"^interval", re.IGNORECASE), types.Interval(), GenericDataType.TEMPORAL, ), - (re.compile(r"^time", re.IGNORECASE), types.Time(), GenericDataType.TEMPORAL,), ( - re.compile(r"^bool.*", re.IGNORECASE), + re.compile(r"^bool(ean)?", re.IGNORECASE), types.Boolean(), GenericDataType.BOOLEAN, ), @@ -693,7 +690,6 @@ def df_to_sql( to_sql_kwargs["name"] = table.table if table.schema: - # Only add schema when it is preset and non empty. to_sql_kwargs["schema"] = table.schema @@ -844,6 +840,7 @@ def get_table_names( # pylint: disable=unused-argument """ Get all tables from schema + :param database: The database to get info :param inspector: SqlAlchemy inspector :param schema: Schema to inspect. If omitted, uses default schema for database :return: All tables in schema @@ -860,6 +857,7 @@ def get_view_names( # pylint: disable=unused-argument """ Get all views from schema + :param database: The database to get info :param inspector: SqlAlchemy inspector :param schema: Schema name. If omitted, uses default schema for database :return: All views in schema @@ -924,7 +922,7 @@ def where_latest_partition( # pylint: disable=too-many-arguments,unused-argumen :param database: Database instance :param query: SqlAlchemy query :param columns: List of TableColumns - :return: SqlAlchemy query with additional where clause referencing latest + :return: SqlAlchemy query with additional where clause referencing the latest partition """ # TODO: Fix circular import caused by importing Database, TableColumn @@ -954,12 +952,12 @@ def select_star( # pylint: disable=too-many-arguments,too-many-locals :param database: Database instance :param table_name: Table name, unquoted - :param engine: SqlALchemy Engine instance + :param engine: SqlAlchemy Engine instance :param schema: Schema, unquoted :param limit: limit to impose on query :param show_cols: Show columns in query; otherwise use "*" :param indent: Add indentation to query - :param latest_partition: Only query latest partition + :param latest_partition: Only query the latest partition :param cols: Columns to include in query :return: SQL query """ @@ -993,7 +991,7 @@ def select_star( # pylint: disable=too-many-arguments,too-many-locals return sql @classmethod - def estimate_statement_cost(cls, statement: str, cursor: Any,) -> Dict[str, Any]: + def estimate_statement_cost(cls, statement: str, cursor: Any) -> Dict[str, Any]: """ Generate a SQL query that estimates the cost of a given statement. @@ -1024,7 +1022,7 @@ def process_statement( :param statement: A single SQL statement :param database: Database instance - :param username: Effective username + :param user_name: Effective username :return: Dictionary with different costs """ parsed_query = ParsedQuery(statement) @@ -1089,7 +1087,6 @@ def update_impersonation_config( :param connect_args: config to be updated :param uri: URI - :param impersonate_user: Flag indicating if impersonation is enabled :param username: Effective username :return: None """ @@ -1122,8 +1119,8 @@ def make_label_compatible(cls, label: str) -> Union[str, quoted_name]: Conditionally mutate and/or quote a sqlalchemy expression label. If force_column_alias_quotes is set to True, return the label as a sqlalchemy.sql.elements.quoted_name object to ensure that the select query - and query results have same case. Otherwise return the mutated label as a - regular string. If maxmimum supported column name length is exceeded, + and query results have same case. Otherwise, return the mutated label as a + regular string. If maximum supported column name length is exceeded, generate a truncated label by calling truncate_label(). :param label: expected expression label/alias @@ -1143,15 +1140,8 @@ def make_label_compatible(cls, label: str) -> Union[str, quoted_name]: def get_sqla_column_type( cls, column_type: Optional[str], - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = column_type_mappings, - ) -> Union[Tuple[TypeEngine, GenericDataType], None]: + column_type_mappings: Tuple[ColumnTypeMapping, ...] = column_type_mappings, + ) -> Optional[Tuple[TypeEngine, GenericDataType]]: """ Return a sqlalchemy native column type that corresponds to the column type defined in the data source (return None to use default type inferred by @@ -1159,16 +1149,18 @@ def get_sqla_column_type( (see MSSQL for example of NCHAR/NVARCHAR handling). :param column_type: Column type returned by inspector + :param column_type_mappings: Maps from string to SqlAlchemy TypeEngine :return: SqlAlchemy column type """ if not column_type: return None for regex, sqla_type, generic_type in column_type_mappings: match = regex.match(column_type) - if match: - if callable(sqla_type): - return sqla_type(match), generic_type - return sqla_type, generic_type + if not match: + continue + if callable(sqla_type): + return sqla_type(match), generic_type + return sqla_type, generic_type return None @staticmethod @@ -1192,7 +1184,7 @@ def _truncate_label(cls, label: str) -> str: """ In the case that a label exceeds the max length supported by the engine, this method is used to construct a deterministic and unique label based on - the original label. By default this returns an md5 hash of the original label, + the original label. By default, this returns a md5 hash of the original label, conditionally truncated if the length of the hash exceeds the max column length of the engine. @@ -1211,8 +1203,8 @@ def column_datatype_to_string( ) -> str: """ Convert sqlalchemy column type to string representation. - By default removes collation and character encoding info to avoid unnecessarily - long datatypes. + By default, removes collation and character encoding info to avoid + unnecessarily long datatypes. :param sqla_column_type: SqlAlchemy column type :param dialect: Sqlalchemy dialect @@ -1304,20 +1296,14 @@ def get_column_spec( # pylint: disable=unused-argument native_type: Optional[str], db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = column_type_mappings, - ) -> Union[ColumnSpec, None]: + column_type_mappings: Tuple[ColumnTypeMapping, ...] = column_type_mappings, + ) -> Optional[ColumnSpec]: """ Converts native database type to sqlalchemy column type. - :param native_type: Native database typee - :param source: Type coming from the database table or cursor description + :param native_type: Native database type :param db_extra: The database extra object + :param source: Type coming from the database table or cursor description + :param column_type_mappings: Maps from string to SqlAlchemy TypeEngine :return: ColumnSpec object """ col_types = cls.get_sqla_column_type( @@ -1417,7 +1403,6 @@ class BasicParametersType(TypedDict, total=False): class BasicParametersMixin: - """ Mixin for configuring DB engine specs via a dictionary. diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 0d7493c386e2a..90cbe621faf56 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -16,7 +16,7 @@ # under the License. import re from datetime import datetime -from typing import Any, Callable, Dict, Match, Optional, Pattern, Tuple, Union +from typing import Any, Dict, Optional, Pattern, Tuple from urllib import parse from flask_babel import gettext as __ @@ -33,9 +33,12 @@ TINYTEXT, ) from sqlalchemy.engine.url import URL -from sqlalchemy.types import TypeEngine -from superset.db_engine_specs.base import BaseEngineSpec, BasicParametersMixin +from superset.db_engine_specs.base import ( + BaseEngineSpec, + BasicParametersMixin, + ColumnTypeMapping, +) from superset.errors import SupersetErrorType from superset.models.sql_lab import Query from superset.utils import core as utils @@ -70,14 +73,7 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin): ) encryption_parameters = {"ssl": "1"} - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = ( + column_type_mappings = ( (re.compile(r"^int.*", re.IGNORECASE), INTEGER(), GenericDataType.NUMERIC,), (re.compile(r"^tinyint", re.IGNORECASE), TINYINT(), GenericDataType.NUMERIC,), ( @@ -208,15 +204,8 @@ def get_column_spec( native_type: Optional[str], db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = column_type_mappings, - ) -> Union[ColumnSpec, None]: + column_type_mappings: Tuple[ColumnTypeMapping, ...] = column_type_mappings, + ) -> Optional[ColumnSpec]: column_spec = super().get_column_spec(native_type) if column_spec: diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index c2951c394d01c..f6c6888ee97bb 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -18,25 +18,18 @@ import logging import re from datetime import datetime -from typing import ( - Any, - Callable, - Dict, - List, - Match, - Optional, - Pattern, - Tuple, - TYPE_CHECKING, - Union, -) +from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING from flask_babel import gettext as __ from sqlalchemy.dialects.postgresql import ARRAY, DOUBLE_PRECISION, ENUM, JSON from sqlalchemy.dialects.postgresql.base import PGInspector -from sqlalchemy.types import String, TypeEngine +from sqlalchemy.types import String -from superset.db_engine_specs.base import BaseEngineSpec, BasicParametersMixin +from superset.db_engine_specs.base import ( + BaseEngineSpec, + BasicParametersMixin, + ColumnTypeMapping, +) from superset.errors import SupersetErrorType from superset.exceptions import SupersetException from superset.models.sql_lab import Query @@ -193,10 +186,10 @@ class PostgresEngineSpec(PostgresBaseEngineSpec, BasicParametersMixin): ( re.compile(r"^array.*", re.IGNORECASE), lambda match: ARRAY(int(match[2])) if match[2] else String(), - utils.GenericDataType.STRING, + GenericDataType.STRING, ), - (re.compile(r"^json.*", re.IGNORECASE), JSON(), utils.GenericDataType.STRING,), - (re.compile(r"^enum.*", re.IGNORECASE), ENUM(), utils.GenericDataType.STRING,), + (re.compile(r"^json.*", re.IGNORECASE), JSON(), GenericDataType.STRING,), + (re.compile(r"^enum.*", re.IGNORECASE), ENUM(), GenericDataType.STRING,), ) @classmethod @@ -275,15 +268,8 @@ def get_column_spec( native_type: Optional[str], db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = column_type_mappings, - ) -> Union[ColumnSpec, None]: + column_type_mappings: Tuple[ColumnTypeMapping, ...] = column_type_mappings, + ) -> Optional[ColumnSpec]: column_spec = super().get_column_spec(native_type) if column_spec: diff --git a/superset/db_engine_specs/presto.py b/superset/db_engine_specs/presto.py index 7f8405220d060..376151587cdee 100644 --- a/superset/db_engine_specs/presto.py +++ b/superset/db_engine_specs/presto.py @@ -23,19 +23,7 @@ from contextlib import closing from datetime import datetime from distutils.version import StrictVersion -from typing import ( - Any, - Callable, - cast, - Dict, - List, - Match, - Optional, - Pattern, - Tuple, - TYPE_CHECKING, - Union, -) +from typing import Any, cast, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING, Union from urllib import parse import pandas as pd @@ -49,11 +37,10 @@ from sqlalchemy.engine.url import make_url, URL from sqlalchemy.orm import Session from sqlalchemy.sql.expression import ColumnClause, Select -from sqlalchemy.types import TypeEngine from superset import cache_manager, is_feature_enabled from superset.common.db_query_status import QueryStatus -from superset.db_engine_specs.base import BaseEngineSpec +from superset.db_engine_specs.base import BaseEngineSpec, ColumnTypeMapping from superset.errors import SupersetErrorType from superset.exceptions import SupersetTemplateException from superset.models.sql_lab import Query @@ -95,7 +82,6 @@ r"line (?P.+?): Catalog '(?P.+?)' does not exist" ) - logger = logging.getLogger(__name__) @@ -449,86 +435,82 @@ def _show_columns( ( re.compile(r"^boolean.*", re.IGNORECASE), types.BOOLEAN, - utils.GenericDataType.BOOLEAN, + GenericDataType.BOOLEAN, ), ( re.compile(r"^tinyint.*", re.IGNORECASE), TinyInteger(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^smallint.*", re.IGNORECASE), types.SMALLINT(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^integer.*", re.IGNORECASE), types.INTEGER(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^bigint.*", re.IGNORECASE), types.BIGINT(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^real.*", re.IGNORECASE), types.FLOAT(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^double.*", re.IGNORECASE), types.FLOAT(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^decimal.*", re.IGNORECASE), types.DECIMAL(), - utils.GenericDataType.NUMERIC, + GenericDataType.NUMERIC, ), ( re.compile(r"^varchar(\((\d+)\))*$", re.IGNORECASE), lambda match: types.VARCHAR(int(match[2])) if match[2] else types.String(), - utils.GenericDataType.STRING, + GenericDataType.STRING, ), ( re.compile(r"^char(\((\d+)\))*$", re.IGNORECASE), lambda match: types.CHAR(int(match[2])) if match[2] else types.CHAR(), - utils.GenericDataType.STRING, + GenericDataType.STRING, ), ( re.compile(r"^varbinary.*", re.IGNORECASE), types.VARBINARY(), - utils.GenericDataType.STRING, - ), - ( - re.compile(r"^json.*", re.IGNORECASE), - types.JSON(), - utils.GenericDataType.STRING, + GenericDataType.STRING, ), + (re.compile(r"^json.*", re.IGNORECASE), types.JSON(), GenericDataType.STRING,), ( re.compile(r"^date.*", re.IGNORECASE), types.DATETIME(), - utils.GenericDataType.TEMPORAL, + GenericDataType.TEMPORAL, ), ( re.compile(r"^timestamp.*", re.IGNORECASE), types.TIMESTAMP(), - utils.GenericDataType.TEMPORAL, + GenericDataType.TEMPORAL, ), ( re.compile(r"^interval.*", re.IGNORECASE), Interval(), - utils.GenericDataType.TEMPORAL, + GenericDataType.TEMPORAL, ), ( re.compile(r"^time.*", re.IGNORECASE), types.Time(), - utils.GenericDataType.TEMPORAL, + GenericDataType.TEMPORAL, ), - (re.compile(r"^array.*", re.IGNORECASE), Array(), utils.GenericDataType.STRING), - (re.compile(r"^map.*", re.IGNORECASE), Map(), utils.GenericDataType.STRING), - (re.compile(r"^row.*", re.IGNORECASE), Row(), utils.GenericDataType.STRING), + (re.compile(r"^array.*", re.IGNORECASE), Array(), GenericDataType.STRING), + (re.compile(r"^map.*", re.IGNORECASE), Map(), GenericDataType.STRING), + (re.compile(r"^row.*", re.IGNORECASE), Row(), GenericDataType.STRING), ) @classmethod @@ -1217,15 +1199,8 @@ def get_column_spec( native_type: Optional[str], db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, - column_type_mappings: Tuple[ - Tuple[ - Pattern[str], - Union[TypeEngine, Callable[[Match[str]], TypeEngine]], - GenericDataType, - ], - ..., - ] = column_type_mappings, - ) -> Union[ColumnSpec, None]: + column_type_mappings: Tuple[ColumnTypeMapping, ...] = column_type_mappings, + ) -> Optional[ColumnSpec]: column_spec = super().get_column_spec( native_type, column_type_mappings=column_type_mappings From 63e2653d33249a63d97e9c78432a5a961f583bb3 Mon Sep 17 00:00:00 2001 From: AAfghahi <48933336+AAfghahi@users.noreply.github.com> Date: Fri, 14 Jan 2022 10:22:30 -0500 Subject: [PATCH 19/20] important change to MakeFile (#18037) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e253d37786f46..47d0f5b55824a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ superset: # Create an admin user in your metadata database superset fab create-admin \ --username admin \ - --firstname Admin \ + --firstname "Admin I."\ --lastname Strator \ --email admin@superset.io \ --password general From 19e2f7bf48495c935ca9f4dd3997e450ac456a1f Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Fri, 21 Jan 2022 13:47:54 +0200 Subject: [PATCH 20/20] add missing is_timeseries to pivot op --- .../test/utils/operators/resampleOperator.test.ts | 3 ++- .../plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts | 1 + superset/utils/pandas_postprocessing.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts index 276616c0db355..a562dbb7f6219 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/operators/resampleOperator.test.ts @@ -94,9 +94,10 @@ test('should do resample on x-axis', () => { ).toEqual({ operation: 'resample', options: { + fill_value: null, + groupby_columns: [], method: 'ffill', rule: '1D', - fill_value: null, time_column: 'ds', }, }); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts index 5a8e190010ff8..1571c1cdf3899 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/Timeseries/buildQuery.ts @@ -44,6 +44,7 @@ export default function buildQuery(formData: QueryFormData) { pivotOperator(formData, { ...baseQueryObject, index: x_axis, + is_timeseries, }); if ( pivotOperatorInRuntime && diff --git a/superset/utils/pandas_postprocessing.py b/superset/utils/pandas_postprocessing.py index 1c4bd74f0baa5..beef42a36ff26 100644 --- a/superset/utils/pandas_postprocessing.py +++ b/superset/utils/pandas_postprocessing.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=too-many-lines import logging from decimal import Decimal from functools import partial