Moving query module into superset-ui from incubator-superset (apache#48)

* migrate query code from incubator-superset

* Getting tests to pass

- Up build-config version to pick up a fix to eslint-typescript-parser
- Remove usage of default exports in favor of named exports unless the export is the only thing being exported out of a module
- Fixing up a few linting errors

* - Remove DatasourceKey interface in favor of readonly id and type properties on the DatasourceKey class directly.
- Adding tests for DatasourceKey.

package.json

@@ -8,7 +8,7 @@
  "build": "yarn run build:cjs && yarn run build:esm && yarn run build:ts",
  "build:cjs": "NODE_ENV=production beemo babel ./src --out-dir lib/ --minify --workspaces=\"@superset-ui/!(demo|generator-superset)\"",
  "build:esm": "NODE_ENV=production beemo babel ./src --out-dir esm/ --esm --minify --workspaces=\"@superset-ui/!(demo|generator-superset)\"",
- "build:ts": "NODE_ENV=production beemo typescript --workspaces=\"@superset-ui/connection\"",
+ "build:ts": "NODE_ENV=production beemo typescript --workspaces=\"@superset-ui/(connection|chart)\"",
  "lint": "beemo create-config prettier && beemo eslint \"./packages/*/{src,test,storybook}/**/*.{js,jsx,ts,tsx}\"",
  "jest": "beemo jest --color --coverage",
  "postrelease": "lerna run gh-pages",
@@ -35,7 +35,7 @@
  ],
  "license": "Apache-2.0",
  "devDependencies": {
- "@data-ui/build-config": "^0.0.31",
+ "@data-ui/build-config": "^0.0.33",
  "husky": "^1.1.2",
  "lerna": "^3.2.1",
  "lint-staged": "^8.0.4",

packages/superset-ui-chart/src/query/Column.ts

@@ -0,0 +1,24 @@
+export enum ColumnType {
+ DOUBLE = 'DOUBLE',
+ FLOAT = 'FLOAT',
+ INT = 'INT',
+ BIGINT = 'BIGINT',
+ LONG = 'LONG',
+ REAL = 'REAL',
+ NUMERIC = 'NUMERIC',
+ DECIMAL = 'DECIMAL',
+ MONEY = 'MONEY',
+ DATE = 'DATE',
+ TIME = 'TIME',
+ DATETIME = 'DATETIME',
+ VARCHAR = 'VARCHAR',
+ STRING = 'STRING',
+ CHAR = 'CHAR',
+}
+
+// TODO: fill out additional fields of the Column interface
+export interface Column {
+ id: number;
+ type: ColumnType;
+ columnName: string;
+}

packages/superset-ui-chart/src/query/DatasourceKey.ts

@@ -0,0 +1,26 @@
+export enum DatasourceType {
+ Table = 'table',
+ Druid = 'druid',
+}
+
+export class DatasourceKey {
+ readonly id: number;
+ readonly type: DatasourceType;
+
+ constructor(key: string) {
+ const [idStr, typeStr] = key.split('__');
+ this.id = parseInt(idStr, 10);
+ this.type = typeStr === 'table' ? DatasourceType.Table : DatasourceType.Druid;
+ }
+
+ public toString() {
+ return `${this.id}__${this.type}`;
+ }
+
+ public toObject() {
+ return {
+ id: this.id,
+ type: this.type,
+ };
+ }
+}

packages/superset-ui-chart/src/query/FormData.ts

@@ -0,0 +1,32 @@
+import { AdhocMetric, MetricKey } from './Metric';
+
+// Type signature and utility functions for formData shared by all viz types
+// It will be gradually filled out as we build out the query object
+
+// Define mapped type separately to work around a limitation of TypeScript
+// https://github.com/Microsoft/TypeScript/issues/13573
+// The Metrics in formData is either a string or a proper metric. It will be
+// unified into a proper Metric type during buildQuery (see `/query/Metrics.ts`).
+type Metrics = Partial<Record<MetricKey, AdhocMetric | string>>;
+
+type BaseFormData = {
+ datasource: string;
+} & Metrics;
+
+// FormData is either sqla-based or druid-based
+type SqlaFormData = {
+ // FormData uses snake_cased keys.
+ // eslint-disable-next-line camelcase
+ granularity_sqla: string;
+} & BaseFormData;
+
+type DruidFormData = {
+ granularity: string;
+} & BaseFormData;
+
+type FormData = SqlaFormData | DruidFormData;
+export default FormData;
+
+export function getGranularity(formData: FormData): string {
+ return 'granularity_sqla' in formData ? formData.granularity_sqla : formData.granularity;
+}

packages/superset-ui-chart/src/query/Metric.ts

@@ -0,0 +1,96 @@
+import { Column } from './Column';
+import FormData from './FormData';
+
+export const LABEL_MAX_LENGTH = 43;
+
+// Note that the values of MetricKeys are lower_snake_case because they're
+// used as keys of form data jsons.
+export enum MetricKey {
+ METRIC = 'metric',
+ METRICS = 'metrics',
+ PERCENT_METRICS = 'percent_metrics',
+ RIGHT_AXIS_METRIC = 'metric_2',
+ SECONDARY_METRIC = 'secondary_metric',
+ X = 'x',
+ Y = 'y',
+ SIZE = 'size',
+}
+
+export enum Aggregate {
+ AVG = 'AVG',
+ COUNT = 'COUNT ',
+ COUNT_DISTINCT = 'COUNT_DISTINCT',
+ MAX = 'MAX',
+ MIN = 'MIN',
+ SUM = 'SUM',
+}
+
+export enum ExpressionType {
+ SIMPLE = 'SIMPLE',
+ SQL = 'SQL',
+}
+
+interface AdhocMetricSimple {
+ expressionType: ExpressionType.SIMPLE;
+ column: Column;
+ aggregate: Aggregate;
+}
+
+interface AdhocMetricSQL {
+ expressionType: ExpressionType.SQL;
+ sqlExpression: string;
+}
+
+export type AdhocMetric = {
+ label?: string;
+ optionName?: string;
+} & (AdhocMetricSimple | AdhocMetricSQL);
+
+export type Metric = {
+ label: string;
+} & Partial<AdhocMetric>;
+
+export class Metrics {
+ // Use Array to maintain insertion order for metrics that are order sensitive
+ private metrics: Metric[];
+
+ constructor(formData: FormData) {
+ this.metrics = Object.keys(MetricKey)
+ .map(key => formData[MetricKey[key as any] as MetricKey])
+ .filter(metric => metric)
+ .map(metric => {
+ if (typeof metric === 'string') {
+ return { label: metric };
+ }
+
+ // Note we further sanitize the metric label for BigQuery datasources
+ // TODO: move this logic to the client once client has more info on the
+ // the datasource
+ return {
+ ...metric,
+ label: (metric as Metric).label || this.getDefaultLabel(metric as AdhocMetric),
+ };
+ });
+ }
+
+ public getMetrics() {
+ return this.metrics;
+ }
+
+ public getLabels() {
+ return this.metrics.map(m => m.label);
+ }
+
+ private getDefaultLabel(metric: AdhocMetric) {
+ let label: string;
+ if (metric.expressionType === ExpressionType.SIMPLE) {
+ label = `${metric.aggregate}(${metric.column.columnName})`;
+ } else {
+ label = metric.sqlExpression;
+ }
+
+ return label.length <= LABEL_MAX_LENGTH
+ ? label
+ : `${label.substring(0, LABEL_MAX_LENGTH - 3)}...`;
+ }
+}

packages/superset-ui-chart/src/query/buildQueryContext.ts

@@ -0,0 +1,16 @@
+import buildQueryObject, { QueryObject } from './buildQueryObject';
+import { DatasourceKey } from './DatasourceKey';
+import FormData from './FormData';
+
+const WRAP_IN_ARRAY = (baseQueryObject: QueryObject) => [baseQueryObject];
+
+// Note: let TypeScript infer the return type
+export default function buildQueryContext(
+ formData: FormData,
+ buildQuery: (baseQueryObject: QueryObject) => QueryObject[] = WRAP_IN_ARRAY,
+) {
+ return {
+ datasource: new DatasourceKey(formData.datasource).toObject(),
+ queries: buildQuery(buildQueryObject(formData)),
+ };
+}

packages/superset-ui-chart/src/query/buildQueryObject.ts

@@ -0,0 +1,21 @@
+import FormData, { getGranularity } from './FormData';
+import { Metric, Metrics } from './Metric';
+
+// TODO: fill out the rest of the query object
+export interface QueryObject {
+ granularity: string;
+ groupby?: string[];
+ metrics?: Metric[];
+}
+
+// Build the common segments of all query objects (e.g. the granularity field derived from
+// either sql alchemy or druid). The segments specific to each viz type is constructed in the
+// buildQuery method for each viz type (see `wordcloud/buildQuery.ts` for an example).
+// Note the type of the formData argument passed in here is the type of the formData for a
+// specific viz, which is a subtype of the generic formData shared among all viz types.
+export default function buildQueryObject<T extends FormData>(formData: T): QueryObject {
+ return {
+ granularity: getGranularity(formData),
+ metrics: new Metrics(formData).getMetrics(),
+ };
+}

packages/superset-ui-chart/src/query/index.ts

@@ -0,0 +1,3 @@
+// Public API of the query module
+export { default } from './buildQueryContext';
+export { default as FormData } from './FormData';

packages/superset-ui-chart/test/query/DatasourceKey.test.ts

@@ -0,0 +1,18 @@
+import { DatasourceKey } from '../../src/query/DatasourceKey';
+
+describe('DatasourceKey', () => {
+ const tableKey = '5__table';
+ const druidKey = '5__druid';
+
+ it('should handle table data sources', () => {
+ const datasourceKey = new DatasourceKey(tableKey);
+ expect(datasourceKey.toString()).toBe(tableKey);
+ expect(datasourceKey.toObject()).toEqual({ id: 5, type: 'table' });
+ });
+
+ it('should handle druid data sources', () => {
+ const datasourceKey = new DatasourceKey(druidKey);
+ expect(datasourceKey.toString()).toBe(druidKey);
+ expect(datasourceKey.toObject()).toEqual({ id: 5, type: 'druid' });
+ });
+});

packages/superset-ui-chart/test/query/Metric.test.ts

@@ -0,0 +1,105 @@
+import { ColumnType } from '../../src/query/Column';
+import {
+ AdhocMetric,
+ Aggregate,
+ ExpressionType,
+ LABEL_MAX_LENGTH,
+ Metrics,
+} from '../../src/query/Metric';
+
+describe('Metrics', () => {
+ let metrics: Metrics;
+ const formData = {
+ datasource: '5__table',
+ granularity_sqla: 'ds',
+ };
+
+ it('should build metrics for built-in metric keys', () => {
+ metrics = new Metrics({
+ ...formData,
+ metric: 'sum__num',
+ });
+ expect(metrics.getMetrics()).toEqual([{ label: 'sum__num' }]);
+ expect(metrics.getLabels()).toEqual(['sum__num']);
+ });
+
+ it('should build metrics for simple adhoc metrics', () => {
+ const adhocMetric: AdhocMetric = {
+ aggregate: Aggregate.AVG,
+ column: {
+ columnName: 'sum_girls',
+ id: 5,
+ type: ColumnType.BIGINT,
+ },
+ expressionType: ExpressionType.SIMPLE,
+ };
+ metrics = new Metrics({
+ ...formData,
+ metric: adhocMetric,
+ });
+ expect(metrics.getMetrics()).toEqual([
+ {
+ aggregate: 'AVG',
+ column: {
+ columnName: 'sum_girls',
+ id: 5,
+ type: ColumnType.BIGINT,
+ },
+ expressionType: 'SIMPLE',
+ label: 'AVG(sum_girls)',
+ },
+ ]);
+ expect(metrics.getLabels()).toEqual(['AVG(sum_girls)']);
+ });
+
+ it('should build metrics for SQL adhoc metrics', () => {
+ const adhocMetric: AdhocMetric = {
+ expressionType: ExpressionType.SQL,
+ sqlExpression: 'COUNT(sum_girls)',
+ };
+ metrics = new Metrics({
+ ...formData,
+ metric: adhocMetric,
+ });
+ expect(metrics.getMetrics()).toEqual([
+ {
+ expressionType: 'SQL',
+ label: 'COUNT(sum_girls)',
+ sqlExpression: 'COUNT(sum_girls)',
+ },
+ ]);
+ expect(metrics.getLabels()).toEqual(['COUNT(sum_girls)']);
+ });
+
+ it('should build metrics for adhoc metrics with custom labels', () => {
+ const adhocMetric: AdhocMetric = {
+ expressionType: ExpressionType.SQL,
+ label: 'foo',
+ sqlExpression: 'COUNT(sum_girls)',
+ };
+ metrics = new Metrics({
+ ...formData,
+ metric: adhocMetric,
+ });
+ expect(metrics.getMetrics()).toEqual([
+ {
+ expressionType: 'SQL',
+ label: 'foo',
+ sqlExpression: 'COUNT(sum_girls)',
+ },
+ ]);
+ expect(metrics.getLabels()).toEqual(['foo']);
+ });
+
+ it('should truncate labels if they are too long', () => {
+ const adhocMetric: AdhocMetric = {
+ expressionType: ExpressionType.SQL,
+ sqlExpression: 'COUNT(verrrrrrrrry_loooooooooooooooooooooong_string)',
+ };
+ metrics = new Metrics({
+ ...formData,
+ metric: adhocMetric,
+ });
+ expect(metrics.getLabels()[0].length).toBeLessThanOrEqual(LABEL_MAX_LENGTH);
+ });
+});

packages/superset-ui-chart/test/query/buildQueryContext.test.ts

@@ -0,0 +1,22 @@
+import build from '../../src/query/buildQueryContext';
+import * as queryObjectBuilder from '../../src/query/buildQueryObject';
+
+describe('queryContextBuilder', () => {
+ it('should build datasource for table sources', () => {
+ const queryContext = build({ datasource: '5__table', granularity_sqla: 'ds' });
+ expect(queryContext.datasource.id).toBe(5);
+ expect(queryContext.datasource.type).toBe('table');
+ });
+
+ it('should build datasource for druid sources', () => {
+ const queryContext = build({ datasource: '5__druid', granularity: 'ds' });
+ expect(queryContext.datasource.id).toBe(5);
+ expect(queryContext.datasource.type).toBe('druid');
+ });
+
+ it('should call queryObjectBuilder to build queries', () => {
+ const buildQueryObjectSpy = jest.spyOn(queryObjectBuilder, 'default');
+ build({ datasource: '5__table', granularity_sqla: 'ds' });
+ expect(buildQueryObjectSpy).toHaveBeenCalledTimes(1);
+ });
+});