Array resolution

meerkat-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.105",
+  "version": "0.0.106",
   "dependencies": {
     "tslib": "^2.3.0",
     "@devrev/meerkat-core": "*",

meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts

@@ -1,18 +1,20 @@
 import {
+  BASE_DATA_SOURCE_NAME,
   ContextParams,
   createBaseTableSchema,
-  generateResolutionJoinPaths,
-  generateResolutionSchemas,
-  generateResolvedDimensions,
+  Dimension,
+  generateRowNumberSql,
+  memberKeyToSafeKey,
   Query,
   ResolutionConfig,
+  ROW_ID_DIMENSION_NAME,
   TableSchema,
 } from '@devrev/meerkat-core';
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
-import {
-  cubeQueryToSQL,
-  CubeQueryToSQLParams,
-} from '../browser-cube-to-sql/browser-cube-to-sql';
+import { cubeQueryToSQL } from '../browser-cube-to-sql/browser-cube-to-sql';
+import { getAggregatedSql } from './steps/aggregation-step';
+import { getResolvedTableSchema } from './steps/resolution-step';
+import { getUnnestTableSchema } from './steps/unnest-step';
 
 export interface CubeQueryToSQLWithResolutionParams {
   connection: AsyncDuckDBConnection;
@@ -38,39 +40,102 @@ export const cubeQueryToSQLWithResolution = async ({
     contextParams,
   });
 
-  if (resolutionConfig.columnConfigs.length === 0) {
-    // If no resolution is needed, return the base SQL.
+  // We have columnProjections check here to ensure that, we are using the same
+  // order in the final query
+  if (
+    resolutionConfig.columnConfigs.length === 0 &&
+    columnProjections?.length === 0
+  ) {
     return baseSql;
   }
 
-  // Create a table schema for the base query.
-  const baseTable: TableSchema = createBaseTableSchema(
+  if (!columnProjections) {
+    columnProjections = [...(query.dimensions || []), ...query.measures];
+  }
+  // This is to ensure that, only the column projection columns
+  // are being resolved and other definitions are ignored.
+  resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter(
+    (config) => {
+      return columnProjections?.includes(config.name);
+    }
+  );
+  return getCubeQueryToSQLWithResolution({
+    connection,
+    baseSql,
+    query,
+    tableSchemas,
+    resolutionConfig,
+    columnProjections,
+    contextParams,
+  });
+};
+
+const getCubeQueryToSQLWithResolution = async ({
+  connection,
+  baseSql,
+  query,
+  tableSchemas,
+  resolutionConfig,
+  columnProjections,
+  contextParams,
+}: {
+  connection: AsyncDuckDBConnection;
+  baseSql: string;
+  query: Query;
+  tableSchemas: TableSchema[];
+  resolutionConfig: ResolutionConfig;
+  columnProjections: string[];
+  contextParams?: ContextParams;
+}): Promise<string> => {
+  const baseSchema: TableSchema = createBaseTableSchema(
     baseSql,
     tableSchemas,
     resolutionConfig,
     query.measures,
     query.dimensions
   );
 
-  const resolutionSchemas: TableSchema[] = generateResolutionSchemas(
+  baseSchema.dimensions.push({
+    name: ROW_ID_DIMENSION_NAME,
+    sql: generateRowNumberSql(
+      query,
+      baseSchema.dimensions,
+      BASE_DATA_SOURCE_NAME
+    ),
+    type: 'number',
+    alias: ROW_ID_DIMENSION_NAME,
+  } as Dimension);
+  columnProjections.push(ROW_ID_DIMENSION_NAME);
+
+  // Doing this because we need to use the original name of the column in the base table schema.
+  resolutionConfig.columnConfigs.forEach((config) => {
+    config.name = memberKeyToSafeKey(config.name);
+  });
+
+  // Generate SQL with row_id and unnested arrays
+  const unnestTableSchema = await getUnnestTableSchema({
+    connection,
+    baseTableSchema: baseSchema,
     resolutionConfig,
-    tableSchemas
-  );
+    contextParams,
+  });
 
-  const resolveParams: CubeQueryToSQLParams = {
-    connection: connection,
-    query: {
-      measures: [],
-      dimensions: generateResolvedDimensions(
-        query,
-        resolutionConfig,
-        columnProjections
-      ),
-      joinPaths: generateResolutionJoinPaths(resolutionConfig, tableSchemas),
-    },
-    tableSchemas: [baseTable, ...resolutionSchemas],
-  };
-  const sql = await cubeQueryToSQL(resolveParams);
+  //  Apply resolution (join with lookup tables)
+  const resolvedTableSchema = await getResolvedTableSchema({
+    connection,
+    baseTableSchema: unnestTableSchema,
+    resolutionConfig,
+    contextParams,
+    columnProjections,
+  });
+
+  // Re-aggregate to reverse the unnest
+  const aggregatedSql = await getAggregatedSql({
+    connection,
+    resolvedTableSchema,
+    resolutionConfig,
+    contextParams,
+  });
 
-  return sql;
+  return aggregatedSql;
 };

meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts

@@ -0,0 +1,114 @@
+import {
+  ContextParams,
+  getArrayTypeResolutionColumnConfigs,
+  getNamespacedKey,
+  Measure,
+  MEERKAT_OUTPUT_DELIMITER,
+  ResolutionConfig,
+  ROW_ID_DIMENSION_NAME,
+  TableSchema,
+  wrapWithRowIdOrderingAndExclusion,
+} from '@devrev/meerkat-core';
+import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
+import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql';
+
+/**
+ * Re-aggregate to reverse the unnest
+ *
+ * This function:
+ * 1. Groups by row_id
+ * 2. Uses MAX for non-array columns (they're duplicated)
+ * 3. Uses ARRAY_AGG for resolved array columns
+ *
+ * @param resolvedTableSchema - Schema from Phase 2 (contains all column info)
+ * @param resolutionConfig - Resolution configuration
+ * @param contextParams - Optional context parameters
+ * @returns Final SQL with arrays containing resolved values
+ */
+export const getAggregatedSql = async ({
+  connection,
+  resolvedTableSchema,
+  resolutionConfig,
+  contextParams,
+}: {
+  connection: AsyncDuckDBConnection;
+  resolvedTableSchema: TableSchema;
+  resolutionConfig: ResolutionConfig;
+  contextParams?: ContextParams;
+}): Promise<string> => {
+  const aggregationBaseTableSchema: TableSchema = resolvedTableSchema;
+
+  // Identify which columns need ARRAY_AGG vs MAX
+  const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig);
+  const baseTableName = aggregationBaseTableSchema.name;
+
+  const isResolvedArrayColumn = (dimName: string) => {
+    return arrayColumns.some((arrayCol) => {
+      return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`);
+    });
+  };
+
+  // Create aggregation measures with proper aggregation functions
+  // Get row_id dimension for GROUP BY
+  const rowIdDimension = aggregationBaseTableSchema.dimensions.find(
+    (d) => d.name === ROW_ID_DIMENSION_NAME
+  );
+
+  if (!rowIdDimension) {
+    throw new Error('Row id dimension not found');
+  }
+  // Create measures with MAX or ARRAY_AGG based on column type
+  const aggregationMeasures: Measure[] = [];
+
+  aggregationBaseTableSchema.dimensions
+    .filter((dim) => dim.name !== rowIdDimension?.name)
+    .forEach((dim) => {
+      const isArrayColumn = isResolvedArrayColumn(dim.name);
+
+      // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id")
+      // We just need to wrap it in the aggregation function
+      const columnRef =
+        dim.sql || `${baseTableName}."${dim.alias || dim.name}"`;
+
+      // Use ARRAY_AGG for resolved array columns, MAX for others
+      // Filter out null values for ARRAY_AGG using FILTER clause
+      const aggregationFn = isArrayColumn
+        ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])`
+        : `MAX(${columnRef})`;
+
+      aggregationMeasures.push({
+        name: dim.name,
+        sql: aggregationFn,
+        type: dim.type,
+        alias: dim.alias,
+      });
+    });
+
+  // Update the schema with aggregation measures
+  const schemaWithAggregation: TableSchema = {
+    ...aggregationBaseTableSchema,
+    measures: aggregationMeasures,
+    dimensions: [rowIdDimension],
+  };
+
+  // Generate the final SQL
+  const aggregatedSql = await cubeQueryToSQL({
+    connection,
+    query: {
+      measures: aggregationMeasures.map((m) =>
+        getNamespacedKey(baseTableName, m.name)
+      ),
+      dimensions: rowIdDimension
+        ? [getNamespacedKey(baseTableName, rowIdDimension.name)]
+        : [],
+    },
+    tableSchemas: [schemaWithAggregation],
+    contextParams,
+  });
+
+  // Order by row_id to maintain consistent ordering before excluding it
+  return wrapWithRowIdOrderingAndExclusion(
+    aggregatedSql,
+    ROW_ID_DIMENSION_NAME
+  );
+};