Merge pull request #1130 from merico-dev/1129-aggregation-by-a-custom…

…-function

1129 aggregation by a custom function

api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devtable/api",
-  "version": "10.18.1",
+  "version": "10.19.0",
   "description": "",
   "main": "index.js",
   "scripts": {

dashboard/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devtable/dashboard",
-  "version": "10.18.1",
+  "version": "10.19.0",
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public",

dashboard/src/components/panel/settings/common/aggregation-selector.tsx

@@ -1,6 +1,9 @@
-import { Group, NumberInput, Select, SpacingValue, Sx, SystemProp } from '@mantine/core';
+import { Group, NumberInput, Select, SpacingValue, SystemProp } from '@mantine/core';
+import { IconMathFunction } from '@tabler/icons-react';
 import React, { useEffect } from 'react';
-import { AggregationType } from '~/utils/aggregation';
+import { InlineFunctionInput } from '~/components/widgets/inline-function-input';
+import { ModalFunctionEditor } from '~/components/widgets/modal-function-editor';
+import { AggregationType, DefaultCustomAggregationFunc } from '~/utils/aggregation';
 
 const options: { label: string; value: AggregationType['type'] }[] = [
   { label: 'None', value: 'none' },
@@ -12,6 +15,7 @@ const options: { label: string; value: AggregationType['type'] }[] = [
   { label: 'Coefficient of Variation', value: 'CV' },
   { label: 'Standard Variation', value: 'std' },
   { label: 'Quantile(99%, 95%, ...)', value: 'quantile' },
+  { label: 'Custom', value: 'custom' },
 ];
 
 interface IAggregationSelector {
@@ -25,7 +29,6 @@ function _AggregationSelector({ label, value, onChange, pt = 'sm' }: IAggregatio
   // migrate from legacy
   useEffect(() => {
     if (typeof value === 'string') {
-      console.log(value);
       onChange({
         type: value,
         config: {},
@@ -36,6 +39,8 @@ function _AggregationSelector({ label, value, onChange, pt = 'sm' }: IAggregatio
   const changeType = (type: AggregationType['type']) => {
     if (type === 'quantile') {
       onChange({ type: 'quantile', config: { p: 0.99 } });
+    } else if (type === 'custom') {
+      onChange({ type: 'custom', config: { func: DefaultCustomAggregationFunc } });
     } else {
       onChange({ type, config: {} });
     }
@@ -49,6 +54,15 @@ function _AggregationSelector({ label, value, onChange, pt = 'sm' }: IAggregatio
       },
     });
   };
+
+  const changeCustomFunc = (func: TFunctionString) => {
+    onChange({
+      type: 'custom',
+      config: {
+        func,
+      },
+    });
+  };
   return (
     <Group grow noWrap pt={pt}>
       <Select ref={ref} label={label} data={options} value={value.type} onChange={changeType} />
@@ -63,6 +77,20 @@ function _AggregationSelector({ label, value, onChange, pt = 'sm' }: IAggregatio
           max={1}
         />
       )}
+      {value.type === 'custom' && (
+        <ModalFunctionEditor
+          label=""
+          triggerLabel="Edit Function"
+          value={value.config.func}
+          onChange={changeCustomFunc}
+          defaultValue={DefaultCustomAggregationFunc}
+          triggerButtonProps={{
+            size: 'xs',
+            sx: { flexGrow: 0, alignSelf: 'center', marginTop: '22px' },
+            leftIcon: <IconMathFunction size={16} />,
+          }}
+        />
+      )}
     </Group>
   );
 }

dashboard/src/components/plugins/viz-components/cartesian/editors/y-axes.tsx

@@ -35,7 +35,7 @@ function YAxisField({ control, index, remove }: IYAxisField) {
           control={control}
           render={({ field }) => (
             // @ts-expect-error type of onChange
-            <Select label="Align" required data={nameAlignmentOptions} sx={{ flex: 1 }} {...field} />
+            <Select label="Name Anchor" required data={nameAlignmentOptions} sx={{ flex: 1 }} {...field} />
           )}
         />
       </Group>

dashboard/src/components/plugins/viz-components/cartesian/option/series/reference_lines.ts

@@ -23,7 +23,7 @@ export function getReferenceLines(
         data: [
           {
             name: r.name,
-            [keyOfAxis]: Number(variableValueMap[r.variable_key]),
+            [keyOfAxis]: variableValueMap[r.variable_key],
           },
         ],
         silent: true,

dashboard/src/components/widgets/about-function-utils/index.tsx

@@ -9,7 +9,7 @@ export function AboutFunctionUtils() {
 
   return (
     <>
-      <Modal opened={opened} onClose={close} title="About FunctionUtils" zIndex={320}>
+      <Modal opened={opened} onClose={close} title="About FunctionUtils" zIndex={330} withinPortal>
         <ReadonlyRichText
           value={FunctionUtilsDescription}
           styles={{ root: { border: 'none' }, content: { padding: 0, table: { marginBottom: 0 } } }}

dashboard/src/components/widgets/modal-function-editor/index.tsx

@@ -0,0 +1,34 @@
+import { Box, Button, Group, Modal } from '@mantine/core';
+import { useDisclosure } from '@mantine/hooks';
+import { forwardRef } from 'react';
+import { InlineFunctionInput } from '../inline-function-input';
+import { AnyObject } from '~/types';
+
+interface Props {
+  value: TFunctionString;
+  onChange: (v: TFunctionString) => void;
+  defaultValue: TFunctionString;
+  label: string;
+  triggerLabel?: string;
+  triggerButtonProps?: AnyObject;
+}
+
+export const ModalFunctionEditor = forwardRef(
+  ({ value, onChange, label, triggerLabel = 'Edit', triggerButtonProps = {}, defaultValue }: Props, _ref: any) => {
+    const [opened, { open, close }] = useDisclosure(false);
+
+    return (
+      <>
+        <Modal opened={opened} onClose={close} title="Authentication" withinPortal zIndex={320} size="900px">
+          <Box h={600}>
+            <InlineFunctionInput value={value} onChange={onChange} defaultValue={defaultValue} label={label} />
+          </Box>
+        </Modal>
+
+        <Button onClick={open} {...triggerButtonProps}>
+          {triggerLabel}
+        </Button>
+      </>
+    );
+  },
+);

dashboard/src/dashboard-editor/ui/settings/content/edit-panel/variable-config/variable-field.tsx

@@ -47,9 +47,12 @@ export const TemplateVariableField = React.forwardRef(function _TemplateVariable
         value={value.aggregation}
         onChange={(v) => handleChange('aggregation', v)}
       />
-
-      <Divider my="xs" label="Format" labelPosition="center" />
-      <NumbroFormatSelector value={value.formatter} onChange={(v) => handleChange('formatter', v)} />
+      {value.aggregation.type !== 'custom' && (
+        <>
+          <Divider my="xs" label="Format" labelPosition="center" />
+          <NumbroFormatSelector value={value.formatter} onChange={(v) => handleChange('formatter', v)} />
+        </>
+      )}
 
       {withStyle && <TemplateVariableStyleField value={value} onChange={onChange} />}
     </Box>

dashboard/src/model/meta-model/dashboard/content/panel/variable.ts

@@ -32,6 +32,10 @@ export const VariableMeta = types
         type: types.literal('quantile'),
         config: types.model({ p: types.number }),
       }),
+      types.model({
+        type: types.literal('custom'),
+        config: types.model({ func: types.string }),
+      }),
     ),
   })
   .views((self) => ({

dashboard/src/utils/aggregation.ts

@@ -1,7 +1,20 @@
 import { quantile } from 'd3-array';
 import _ from 'lodash';
 import * as math from 'mathjs';
-import { extractData } from './data';
+import { extractData, extractFullQueryData } from './data';
+import { functionUtils } from './function-utils';
+
+export type TCustomAggregation = {
+  type: 'custom';
+  config: {
+    func: string;
+  };
+};
+export const DefaultCustomAggregationFunc = [
+  'function aggregation({ queryData }, utils) {',
+  '    return "Aggregation Result";',
+  '}',
+].join('\n');
 
 export type AggregationType =
   | {
@@ -13,7 +26,8 @@ export type AggregationType =
       config: {
         p: number;
       };
-    };
+    }
+  | TCustomAggregation;
 export const DefaultAggregation: AggregationType = {
   type: 'none',
   config: {},
@@ -71,8 +85,21 @@ export function formatNumbersAndAggregateValue(possibleNumbers: Array<string | n
   return aggregateValueFromNumbers(numbers, aggregation);
 }
 
+function runCustomAggregation(data: TPanelData, data_field: string, aggregation: TCustomAggregation) {
+  try {
+    const queryData = extractFullQueryData(data, data_field);
+    return new Function(`return ${aggregation.config.func}`)()({ queryData }, functionUtils);
+  } catch (error) {
+    console.error(error);
+    return (error as any).message;
+  }
+}
+
 export function aggregateValue(data: TPanelData, data_field: string, aggregation: AggregationType) {
   try {
+    if (aggregation.type === 'custom') {
+      return runCustomAggregation(data, data_field, aggregation);
+    }
     return formatNumbersAndAggregateValue(extractData(data, data_field), aggregation);
   } catch (error) {
     console.error(error);

dashboard/src/utils/diff-json.ts

@@ -0,0 +1,21 @@
+import _ from 'lodash';
+import { AnyObject } from '..';
+
+function printChanges(a: any, b: any) {
+  if (_.isEqual(a, b)) {
+    return;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    a.forEach((_a, i) => {
+      printChanges(_a, b[i]);
+    });
+    return;
+  }
+  console.log(a, b);
+}
+
+export function printJSONChanges(a: AnyObject, b: AnyObject) {
+  Object.keys(a).forEach((k) => {
+    printChanges(a[k], b[k]);
+  });
+}

dashboard/src/utils/template/editor/variable-field.tsx

@@ -52,7 +52,9 @@ export const TemplateVariableField = React.forwardRef(function _TemplateVariable
       />
 
       <Divider my="xs" label="Format" labelPosition="center" />
-      <NumbroFormatSelector value={value.formatter} onChange={(v) => handleChange('formatter', v)} />
+      {value.aggregation.type !== 'custom' && (
+        <NumbroFormatSelector value={value.formatter} onChange={(v) => handleChange('formatter', v)} />
+      )}
 
       {withStyle && <TemplateVariableStyleField value={value} onChange={onChange} />}
     </Box>

dashboard/src/utils/template/utils.ts

@@ -19,10 +19,16 @@ export function getAggregatedValue({ data_field, aggregation }: ITemplateVariabl
   return aggregateValue(data, data_field, aggregation);
 }
 
-export function formatAggregatedValue({ formatter }: ITemplateVariable, value: number | string | number[] | null) {
+export function formatAggregatedValue(
+  { formatter, aggregation }: ITemplateVariable,
+  value: number | string | number[] | null,
+) {
   if (!['string', 'number'].includes(typeof value)) {
     return getNonStatsDataText(value);
   }
+  if (aggregation.type === 'custom') {
+    return value;
+  }
   try {
     return numbro(value).format(formatter);
   } catch (e) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devtable/root",
-  "version": "10.18.1",
+  "version": "10.19.0",
   "private": true,
   "workspaces": [
     "api",

settings-form/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devtable/settings-form",
-  "version": "10.18.1",
+  "version": "10.19.0",
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public",

website/package.json

@@ -2,7 +2,7 @@
   "name": "@devtable/website",
   "private": true,
   "license": "Apache-2.0",
-  "version": "10.18.1",
+  "version": "10.19.0",
   "scripts": {
     "dev": "vite",
     "preview": "vite preview"