datacube: initial support for pivot() (#3526)

* datacube: fix grid over-triggering change detection * datacube: re-compile active column editor for new snapshot * datacube: add base support for pivot() * datacube: properly sequence pivot function with groupBy * datacube: improve styling of pivot columns * datacube: setup cast columns collection * datacube: fix various compositional bug with pivot and extend * datacube: fix issue with re-arrange columns with active pivot * datacube: add h-pivot support in grid context menu
finos · Sep 17, 2024 · 9cb912c · 9cb912c
1 parent 8dec11e
commit 9cb912c
Show file tree

Hide file tree

Showing 38 changed files with 1,260 additions and 682 deletions.
diff --git a/.changeset/odd-impalas-march.md b/.changeset/odd-impalas-march.md
@@ -0,0 +1,4 @@
+---
+'@finos/legend-application-repl': patch
+'@finos/legend-graph': patch
+---
diff --git a/packages/legend-application-repl/src/components/dataCube/editor/DataCubeEditor.tsx b/packages/legend-application-repl/src/components/dataCube/editor/DataCubeEditor.tsx
@@ -21,7 +21,6 @@ import { DataCubeEditorGeneralPropertiesPanel } from './DataCubeEditorGeneralPro
 import { DataCubeEditorColumnsPanel } from './DataCubeEditorColumnsPanel.js';
 import { DataCubeEditorVerticalPivotsPanel } from './DataCubeEditorVerticalPivotsPanel.js';
 import { DataCubeEditorHorizontalPivotsPanel } from './DataCubeEditorHorizontalPivotsPanel.js';
-import { DataCubeEditorCodePanel } from './DataCubeEditorCodePanel.js';
 import { DataCubeEditorColumnPropertiesPanel } from './DataCubeEditorColumnPropertiesPanel.js';
 import { cn } from '@finos/legend-art';
 import type { DataCubeState } from '../../../stores/dataCube/DataCubeState.js';
@@ -39,7 +38,6 @@ export const DataCubeEditor = observer((props: { dataCube: DataCubeState }) => {
     DataCubeEditorTab.SORTS,
     DataCubeEditorTab.GENERAL_PROPERTIES,
     DataCubeEditorTab.COLUMN_PROPERTIES,
-    DataCubeEditorTab.CODE,
   ];
 
   return (
@@ -80,14 +78,11 @@ export const DataCubeEditor = observer((props: { dataCube: DataCubeState }) => {
           {selectedTab === DataCubeEditorTab.COLUMN_PROPERTIES && (
             <DataCubeEditorColumnPropertiesPanel dataCube={dataCube} />
           )}
-          {selectedTab === DataCubeEditorTab.CODE && (
-            <DataCubeEditorCodePanel dataCube={dataCube} />
-          )}
         </div>
       </div>
       <div className="flex h-10 items-center justify-end px-2">
         <button
-          className="h-6 w-20 border border-neutral-400 bg-neutral-300 px-2 hover:brightness-95"
+          className="h-6 w-20 border border-neutral-400 bg-neutral-300 px-2 hover:brightness-95 disabled:cursor-not-allowed disabled:border-neutral-300 disabled:text-neutral-400 disabled:hover:brightness-100"
           disabled={editor.finalizationState.isInProgress}
           onClick={() => {
             editor
@@ -104,7 +99,7 @@ export const DataCubeEditor = observer((props: { dataCube: DataCubeState }) => {
           Cancel
         </button>
         <button
-          className="ml-2 h-6 w-20 border border-neutral-400 bg-neutral-300 px-2 hover:brightness-95"
+          className="ml-2 h-6 w-20 border border-neutral-400 bg-neutral-300 px-2 hover:brightness-95 disabled:cursor-not-allowed disabled:border-neutral-300 disabled:text-neutral-400 disabled:hover:brightness-100"
           disabled={editor.finalizationState.isInProgress}
           onClick={() => {
             editor.applyChanges().catch(application.alertUnhandledError);

diff --git a/packages/legend-application-repl/src/components/dataCube/editor/DataCubeEditorCodePanel.tsx b/packages/legend-application-repl/src/components/dataCube/editor/DataCubeEditorCodePanel.tsx
diff --git a/...ges/legend-application-repl/src/components/dataCube/editor/DataCubeEditorColumnsPanel.tsx b/...ges/legend-application-repl/src/components/dataCube/editor/DataCubeEditorColumnsPanel.tsx
@@ -78,7 +78,7 @@ export const DataCubeEditorColumnsPanel = observer(
     const { dataCube } = props;
     const panel = dataCube.editor.columns;
 
-    useEffect(() => () => panel.propagateColumnSelectionChanges(), [panel]);
+    useEffect(() => () => panel.propgateChanges(), [panel]);
 
     return (
       <div className="h-full w-full select-none p-2">

diff --git a/...d-application-repl/src/components/dataCube/editor/DataCubeEditorHorizontalPivotsPanel.tsx b/...d-application-repl/src/components/dataCube/editor/DataCubeEditorHorizontalPivotsPanel.tsx
@@ -18,12 +18,25 @@ import { DataCubeIcon } from '@finos/legend-art';
 import { observer } from 'mobx-react-lite';
 import type { DataCubeState } from '../../../stores/dataCube/DataCubeState.js';
 import { DataCubeEditorColumnsSelector } from './DataCubeEditorColumnsSelector.js';
+import { useEffect } from 'react';
+import { FormBadge_WIP } from '../../repl/Form.js';
+
+const PivotColumnSortDirectionDropdown = observer((props) => (
+  <div className="relative flex h-full items-center">
+    <div className="flex h-[18px] w-32 items-center border border-transparent px-2 text-sm text-neutral-400">
+      Ascending
+      <FormBadge_WIP />
+    </div>
+  </div>
+));
 
 export const DataCubeEditorHorizontalPivotsPanel = observer(
   (props: { dataCube: DataCubeState }) => {
     const { dataCube } = props;
     const panel = dataCube.editor.horizontalPivots;
 
+    useEffect(() => () => panel.propagateChanges(), [panel]);
+
     return (
       <div className="h-full w-full select-none p-2">
         <div className="flex h-6">
@@ -35,7 +48,12 @@ export const DataCubeEditorHorizontalPivotsPanel = observer(
           </div>
         </div>
         <div className="flex h-[calc(100%_-_24px)] w-full">
-          <DataCubeEditorColumnsSelector selector={panel.selector} />
+          <DataCubeEditorColumnsSelector
+            selector={panel.selector}
+            columnActionRenderer={(p) => (
+              <PivotColumnSortDirectionDropdown {...p} />
+            )}
+          />
         </div>
       </div>
     );

diff --git a/packages/legend-application-repl/src/components/dataCube/extend/DataCubeColumnEditor.tsx b/packages/legend-application-repl/src/components/dataCube/extend/DataCubeColumnEditor.tsx
@@ -341,7 +341,8 @@ export const DataCubeColumnCreator = observer(
             disabled={
               !state.isNameValid ||
               !state.isTypeValid ||
-              !state.validationState.hasCompleted
+              state.validationState.isInProgress ||
+              state.finalizationState.isInProgress
             }
             onClick={() => {
               state.applyChanges().catch(application.alertUnhandledError);

diff --git a/packages/legend-application-repl/src/components/dataCube/grid/DataCubeGrid.tsx b/packages/legend-application-repl/src/components/dataCube/grid/DataCubeGrid.tsx
@@ -281,7 +281,7 @@ const DataCubeGridStatusBar = observer((props: { dataCube: DataCubeState }) => {
             ? `Rows: ${grid.clientDataSource.rowCount}`
             : ''}
         </div>
-        {grid.datasourceConfiguration.limit !== undefined &&
+        {grid.rowLimit !== undefined &&
           grid.queryConfiguration.showWarningForTruncatedResult && (
             // TODO: if we want to properly warn if the data has been truncated due to row limit,
             // this would require us to fetch n+1 rows when limit=n
@@ -292,7 +292,7 @@ const DataCubeGridStatusBar = observer((props: { dataCube: DataCubeState }) => {
               <div className="h-3 w-[1px] bg-neutral-200" />
               <div className="flex h-full items-center px-2 text-orange-500">
                 <DataCubeIcon.Warning className="stroke-[3px]" />
-                <div className="ml-1 text-sm font-semibold">{`Results truncated to fit within row limit (${grid.datasourceConfiguration.limit})`}</div>
+                <div className="ml-1 text-sm font-semibold">{`Results truncated to fit within row limit (${grid.rowLimit})`}</div>
               </div>
             </>
           )}

diff --git a/packages/legend-application-repl/src/components/repl/Form.tsx b/packages/legend-application-repl/src/components/repl/Form.tsx
@@ -42,7 +42,7 @@ import { useREPLStore } from '../REPLStoreProvider.js';
 export function FormBadge_WIP() {
   return (
     <div
-      className="text-2xs ml-1 select-none rounded-md bg-sky-500 px-1 py-0.5 font-semibold text-white"
+      className="text-2xs ml-1 flex h-2.5 flex-shrink-0 select-none items-center rounded-md bg-sky-500 px-1 font-semibold leading-[10px] text-white"
       title="Work In Progress"
     >
       WIP

diff --git a/packages/legend-application-repl/src/stores/dataCube/DataCubeEngine.ts b/packages/legend-application-repl/src/stores/dataCube/DataCubeEngine.ts
@@ -15,10 +15,10 @@
  */
 
 import {
+  type V1_ValueSpecification,
   type V1_Lambda,
   V1_deserializeValueSpecification,
   V1_serializeValueSpecification,
-  type V1_ValueSpecification,
   TDSExecutionResult,
   V1_serializeExecutionResult,
   V1_buildExecutionResult,

diff --git a/packages/legend-application-repl/src/stores/dataCube/core/DataCubeConfiguration.ts b/packages/legend-application-repl/src/stores/dataCube/core/DataCubeConfiguration.ts
@@ -90,6 +90,7 @@ export class DataCubeColumnConfiguration {
   displayAsLink = false;
   linkLabelParameter?: string | undefined;
 
+  // NOTE: these configurations, when changed, would potentially trigger data-fetching
   aggregateOperator!: string;
   aggregationParameters: DataCubeOperationValue[] = [];
   excludedFromHorizontalPivot = true; // this agrees with default column kind set as Dimension
@@ -175,22 +176,18 @@ export class DataCubeConfiguration {
   alternateRowsColor = DEFAULT_ROW_HIGHLIGHT_BACKGROUND_COLOR;
   alternateRowsCount = 1;
 
-  // aggregation
+  selectionStats: DataCubeSelectionStat[] = [];
+  showWarningForTruncatedResult = true;
+
+  // NOTE: these configurations, when changed, would potentially trigger data-fetching
   initialExpandLevel?: number | undefined;
   showRootAggregation = false;
   showLeafCount = false;
-  addPivotTotalColumn = true;
-  addPivotTotalColumnOnLeft = true;
+  pivotTotalColumnPlacement?: DataCubeColumnPinPlacement | undefined;
   treeGroupSortFunction?: string | undefined;
 
-  // misc
-  selectionStats: DataCubeSelectionStat[] = [];
-  showWarningForTruncatedResult = true;
-
   static readonly serialization = new SerializationFactory(
     createModelSchema(DataCubeConfiguration, {
-      addPivotTotalColumn: primitive(),
-      addPivotTotalColumnOnLeft: primitive(),
       alternateRows: primitive(),
       alternateRowsColor: primitive(),
       alternateRowsCount: primitive(),
@@ -214,6 +211,7 @@ export class DataCubeConfiguration {
       negativeForegroundColor: primitive(),
       normalBackgroundColor: primitive(),
       normalForegroundColor: primitive(),
+      pivotTotalColumnPlacement: optional(primitive()),
       selectionStats: list(primitive()),
       showHorizontalGridLines: primitive(),
       showLeafCount: primitive(),

diff --git a/packages/legend-application-repl/src/stores/dataCube/core/DataCubeQueryBuilder.ts b/packages/legend-application-repl/src/stores/dataCube/core/DataCubeQueryBuilder.ts
@@ -55,6 +55,8 @@ import {
   _function,
   _groupByAggCols,
   _lambda,
+  _pivotAggCols,
+  _castCols,
   _primitiveValue,
   _var,
 } from './DataCubeQueryBuilderUtils.js';
@@ -103,13 +105,13 @@ export function buildExecutableQuery(
       'leafExtend',
       _function(_name(DataCubeFunction.EXTEND), [
         _cols(
-          data.leafExtendedColumns.map((eCol) => {
-            if (eCol._type === DataCubeExtendedColumnType.SIMPLE) {
-              const col = eCol as DataCubeQuerySnapshotSimpleExtendedColumn;
-              return _colSpec(col.name, _deserializeToLambda(col.lambda));
+          data.leafExtendedColumns.map((col) => {
+            if (col._type === DataCubeExtendedColumnType.SIMPLE) {
+              const column = col as DataCubeQuerySnapshotSimpleExtendedColumn;
+              return _colSpec(column.name, _deserializeToLambda(column.lambda));
             }
             throw new UnsupportedOperationError(
-              `Can't build extended column of type '${eCol._type}'`,
+              `Can't build extended column of type '${col._type}'`,
             );
           }),
         ),
@@ -139,6 +141,46 @@ export function buildExecutableQuery(
     );
   }
 
+  // --------------------------------- PIVOT ---------------------------------
+
+  if (data.pivot) {
+    const pivot = data.pivot;
+
+    // pre-sort to maintain stable order for pivot result columns
+    _process(
+      'sort',
+      _function(_name(DataCubeFunction.SORT), [
+        _collection(
+          data.pivot.columns.map((col) =>
+            _function(_name(DataCubeFunction.ASC), [_col(col.name)]),
+          ),
+        ),
+      ]),
+    );
+
+    _process(
+      'pivot',
+      _function(_name(DataCubeFunction.PIVOT), [
+        _cols(pivot.columns.map((col) => _colSpec(col.name))),
+        _cols(
+          _pivotAggCols(
+            pivot.columns,
+            snapshot,
+            configuration,
+            aggregateOperations,
+          ),
+        ),
+      ]),
+    );
+
+    if (pivot.castColumns.length) {
+      _process(
+        'pivotCast',
+        _function(_name(DataCubeFunction.CAST), [_castCols(pivot.castColumns)]),
+      );
+    }
+  }
+
   // --------------------------------- GROUP BY ---------------------------------
 
   if (data.groupBy) {
@@ -159,23 +201,20 @@ export function buildExecutableQuery(
     );
   }
 
-  // --------------------------------- PIVOT ---------------------------------
-  /** TODO: @datacube pivot - implement this and CAST */
-
   // --------------------------------- GROUP-LEVEL EXTEND ---------------------------------
 
   if (data.groupExtendedColumns.length) {
     _process(
       'groupExtend',
       _function(_name(DataCubeFunction.EXTEND), [
         _cols(
-          data.groupExtendedColumns.map((eCol) => {
-            if (eCol._type === DataCubeExtendedColumnType.SIMPLE) {
-              const col = eCol as DataCubeQuerySnapshotSimpleExtendedColumn;
-              return _colSpec(col.name, _deserializeToLambda(col.lambda));
+          data.groupExtendedColumns.map((col) => {
+            if (col._type === DataCubeExtendedColumnType.SIMPLE) {
+              const column = col as DataCubeQuerySnapshotSimpleExtendedColumn;
+              return _colSpec(column.name, _deserializeToLambda(column.lambda));
             }
             throw new UnsupportedOperationError(
-              `Can't build extended column of type '${eCol._type}'`,
+              `Can't build extended column of type '${col._type}'`,
             );
           }),
         ),