refactor(core): move mexp from plugins to core (apache#1371)

* refactor(core): move mexp from plugins to core

* remove test nest

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/package.json

@@ -39,6 +39,7 @@
  "@types/d3-time": "^1.0.9",
  "@types/d3-time-format": "^2.1.0",
  "@types/lodash": "^4.14.149",
+ "@types/math-expression-evaluator": "^1.2.1",
  "@types/rison": "0.0.6",
  "@types/seedrandom": "^2.4.28",
  "@vx/responsive": "^0.0.199",
@@ -51,6 +52,7 @@
  "fetch-retry": "^4.0.1",
  "jed": "^1.1.1",
  "lodash": "^4.17.11",
+ "math-expression-evaluator": "^1.3.8",
  "pretty-ms": "^7.0.0",
  "react-error-boundary": "^1.2.5",
  "react-markdown": "^4.3.1",

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/index.ts

@@ -32,3 +32,4 @@ export * from './validator';
 export * from './chart';
 export * from './chart-composition';
 export * from './components';
+export * from './math-expression';

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/src/math-expression/index.ts

@@ -0,0 +1,102 @@
+import mexp from 'math-expression-evaluator';
+
+const REPLACE_OPERATORS: [RegExp, string][] = [
+ [new RegExp(/==/g), 'Eq'],
+ [new RegExp(/>=/g), 'Gte'],
+ [new RegExp(/<=/g), 'Lte'],
+ [new RegExp(/>/g), 'Gt'],
+ [new RegExp(/</g), 'Lt'],
+];
+
+const TOKENS = [
+ {
+ type: 3,
+ token: 'x',
+ show: 'x',
+ value: 'x',
+ },
+ {
+ type: 2,
+ token: '&',
+ show: '&',
+ value: (a: number, b: number): number => a & b,
+ },
+ {
+ type: 2,
+ token: '|',
+ show: '|',
+ value: (a: number, b: number): number => a | b,
+ },
+ {
+ type: 2,
+ token: 'and',
+ show: 'and',
+ value: (a: number, b: number): number => a && b,
+ },
+ {
+ type: 2,
+ token: 'xor',
+ show: 'xor',
+ value: (a: number, b: number): number => a ^ b,
+ },
+ {
+ type: 2,
+ token: 'or',
+ show: 'or',
+ value: (a: number, b: number): number => Number(a || b),
+ },
+ {
+ type: 2,
+ token: 'Eq',
+ show: 'Eq',
+ value: (a: number, b: number): number => Number(a === b),
+ },
+ {
+ type: 2,
+ token: 'Lt',
+ show: 'Lt',
+ value: (a: number, b: number): number => Number(a < b),
+ },
+ {
+ type: 2,
+ token: 'Lte',
+ show: 'Lte',
+ value: (a: number, b: number): number => Number(a <= b),
+ },
+ {
+ type: 2,
+ token: 'Gt',
+ show: 'Gt',
+ value: (a: number, b: number): number => Number(a > b),
+ },
+ {
+ type: 2,
+ token: 'Gte',
+ show: 'Gte',
+ value: (a: number, b: number): number => Number(a >= b),
+ },
+];
+
+export function evalExpression(expression: string, value: number): number {
+ let parsedExpression = expression;
+ // replace `<` with `Lt` (and others) to avoid clashes with builtin function operators
+ // that are not needed in Superset.
+ REPLACE_OPERATORS.forEach(([key, value]) => {
+ parsedExpression = parsedExpression.replace(key, value);
+ });
+ const subExpressions = String(parsedExpression).split('=');
+ parsedExpression = subExpressions[1] ?? subExpressions[0];
+ // we can ignore the type requirement on `TOKENS`, as value is always `number`
+ // and doesn't need to consider `number | underfined`.
+ // @ts-ignore
+ return Number(mexp.eval(parsedExpression, TOKENS, { x: value }));
+}
+
+export function isValidExpression(expression: string): boolean {
+ try {
+ evalExpression(expression, 0);
+ } catch (err) {
+ return false;
+ }
+ return true;
+}

superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-core/test/math-expression/index.test.ts

@@ -0,0 +1,79 @@
+import { evalExpression, isValidExpression } from '@superset-ui/core/src/math-expression';
+
+test('evalExpression evaluates constants correctly', () => {
+ expect(evalExpression('0', 10)).toEqual(0);
+ expect(evalExpression('0.123456', 0)).toEqual(0.123456);
+ expect(evalExpression('789', 100)).toEqual(789);
+});
+
+test('evalExpression evaluates infinities correctly', () => {
+ const formula = 'x/0';
+ expect(evalExpression(formula, 1)).toEqual(Infinity);
+ expect(evalExpression(formula, -1)).toEqual(-Infinity);
+});
+
+test('evalExpression evaluates powers correctly', () => {
+ const formula = '2^(x/2)*100';
+ expect(evalExpression(formula, 0)).toEqual(100);
+ expect(evalExpression(formula, 1)).toEqual(141.4213562373095);
+ expect(evalExpression(formula, 2)).toEqual(200);
+});
+
+test('evalExpression ignores whitespace and variables on left hand side of equals sign', () => {
+ expect(evalExpression('y=x+1', 1)).toEqual(2);
+ expect(evalExpression('y = x - 1', 1)).toEqual(0);
+});
+
+test('evalExpression evaluates custom operators correctly', () => {
+ const equalsExpression = 'x == 10';
+ expect(evalExpression(equalsExpression, 5)).toEqual(0);
+ expect(evalExpression(equalsExpression, 10)).toEqual(1);
+ expect(evalExpression(equalsExpression, 10.1)).toEqual(0);
+
+ const closedRange = '(x > 0) and (x < 10)';
+ expect(evalExpression(closedRange, 0)).toEqual(0);
+ expect(evalExpression(closedRange, 5)).toEqual(1);
+ expect(evalExpression(closedRange, 10)).toEqual(0);
+
+ const openRange = '(x >= 0) and (x <= 10)';
+ expect(evalExpression(openRange, -0.1)).toEqual(0);
+ expect(evalExpression(openRange, 0)).toEqual(1);
+ expect(evalExpression(openRange, 5)).toEqual(1);
+ expect(evalExpression(openRange, 10)).toEqual(1);
+ expect(evalExpression(openRange, 10.1)).toEqual(0);
+
+ const orRange = '(x < 0) or (x > 10)';
+ expect(evalExpression(orRange, -0.1)).toEqual(1);
+ expect(evalExpression(orRange, 0)).toEqual(0);
+ expect(evalExpression(orRange, 5)).toEqual(0);
+ expect(evalExpression(orRange, 10)).toEqual(0);
+ expect(evalExpression(orRange, 10.1)).toEqual(1);
+
+ // other less used operators
+ expect(evalExpression('5 & x', 3)).toEqual(1);
+ expect(evalExpression('5 | x', 3)).toEqual(7);
+ expect(evalExpression('5 xor x', 2)).toEqual(7);
+
+ // complex combinations
+ const complexExpression = '20.51*(x<1577836800000)+20.2((x<15805152000000)&(x>=1577836800000))';
+ expect(evalExpression(complexExpression, 0)).toEqual(20.51);
+ expect(evalExpression(complexExpression, 1000)).toEqual(20.51);
+ expect(evalExpression(complexExpression, 1577836800000)).toEqual(20.2);
+ expect(evalExpression(complexExpression, 15805151999999)).toEqual(20.2);
+ expect(evalExpression(complexExpression, 15805152000000)).toEqual(0);
+ expect(evalExpression(complexExpression, 15805159000000)).toEqual(0);
+});
+
+test('isValidExpression correctly identifies invalid formulas', () => {
+ expect(isValidExpression('foobar')).toEqual(false);
+ expect(isValidExpression('x+')).toEqual(false);
+ expect(isValidExpression('z+1')).toEqual(false);
+});
+
+test('isValidExpression correctly identifies valid formulas', () => {
+ expect(isValidExpression('x')).toEqual(true);
+ expect(isValidExpression('x+1')).toEqual(true);
+ expect(isValidExpression('y=x-1')).toEqual(true);
+ expect(isValidExpression('y = x - 1')).toEqual(true);
+ expect(isValidExpression('y = (x < 100 and x > 0) * 100')).toEqual(true);
+});

superset-frontend/temporary_superset_ui/superset-ui/plugins/legacy-preset-chart-nvd3/package.json

@@ -36,7 +36,6 @@
  "dompurify": "^2.0.6",
  "fast-safe-stringify": "^2.0.6",
  "lodash": "^4.17.11",
- "math-expression-evaluator": "^1.3.8",
  "moment": "^2.20.1",
  "nvd3-fork": "^2.0.5",
  "prop-types": "^15.6.2",

superset-frontend/temporary_superset_ui/superset-ui/plugins/legacy-preset-chart-nvd3/src/NVD3Vis.js

@@ -20,17 +20,17 @@
 import { kebabCase, throttle } from 'lodash';
 import d3 from 'd3';
 import moment from 'moment';
-import mexp from 'math-expression-evaluator';
 import nv from 'nvd3-fork';
 import PropTypes from 'prop-types';
 import {
- t,
- isDefined,
- getTimeFormatter,
- smartDateVerboseFormatter,
+ CategoricalColorNamespace,
+ evalExpression,
  getNumberFormatter,
+ getTimeFormatter,
+ isDefined,
  NumberFormats,
- CategoricalColorNamespace,
+ smartDateVerboseFormatter,
+ t,
 } from '@superset-ui/core';
 
 import 'nvd3-fork/build/nv.d3.css';
@@ -946,13 +946,6 @@ function nvd3Vis(element, props) {
  }
 
  if (formulas.length > 0) {
- const token = {
- type: 3,
- token: 'x',
- show: 'x',
- value: 'x',
- };
-
  const xValues = [];
  if (vizType === 'bar') {
  // For bar-charts we want one data point evaluated for every
@@ -982,13 +975,11 @@ function nvd3Vis(element, props) {
  }
  const formulaData = formulas.map(fo => {
  const { value: expression } = fo;
- const subExpressions = String(expression).split('=');
-
  return {
  key: fo.name,
  values: xValues.map(x => ({
  x,
- y: mexp.eval(subExpressions[1] ?? subExpressions[0], [token], { x }),
+ y: evalExpression(expression, x),
  })),
  color: fo.color,
  strokeWidth: fo.width,

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-echarts/package.json

@@ -28,11 +28,9 @@
  "dependencies": {
  "@superset-ui/chart-controls": "0.18.5",
  "@superset-ui/core": "0.18.5",
- "@types/math-expression-evaluator": "^1.2.1",
  "d3-array": "^1.2.0",
  "echarts": "^5.2.1",
- "lodash": "^4.17.15",
- "math-expression-evaluator": "^1.3.8"
+ "lodash": "^4.17.15"
  },
  "peerDependencies": {
  "react": "^16.13.1"

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-echarts/src/Timeseries/transformers.ts

@@ -18,11 +18,11 @@
  */
 import {
  AnnotationData,
- AnnotationLayer,
  AnnotationOpacity,
  CategoricalColorScale,
  EventAnnotationLayer,
  FilterState,
+ FormulaAnnotationLayer,
  getTimeFormatter,
  IntervalAnnotationLayer,
  isTimeseriesAnnotationResult,
@@ -204,7 +204,7 @@ export function transformSeries(
 }
 
 export function transformFormulaAnnotation(
- layer: AnnotationLayer,
+ layer: FormulaAnnotationLayer,
  data: TimeseriesDataRecord[],
  colorScale: CategoricalColorScale,
 ): SeriesOption {

superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-echarts/src/utils/annotation.ts

@@ -23,29 +23,23 @@ import {
  AnnotationLayer,
  AnnotationOpacity,
  AnnotationType,
+ evalExpression,
+ FormulaAnnotationLayer,
  isRecordAnnotationResult,
  isTableAnnotationLayer,
  isTimeseriesAnnotationResult,
  TimeseriesDataRecord,
 } from '@superset-ui/core';
-import mexp from 'math-expression-evaluator';
 
 export function evalFormula(
- formula: AnnotationLayer,
+ formula: FormulaAnnotationLayer,
  data: TimeseriesDataRecord[],
 ): [Date, number][] {
- const token = {
- type: 3,
- token: 'x',
- show: 'x',
- value: 'x',
- };
  const { value: expression } = formula;
- const subExpressions = String(expression).split('=');
 
  return data.map(row => [
  new Date(Number(row.__timestamp)),
- Number(mexp.eval(subExpressions[1] ?? subExpressions[0], [token], { x: row.__timestamp })),
+ evalExpression(expression, row.__timestamp as number),
  ]);
 }