feat(schema-compiler): custom granularity support (#8537)

* remove unneeded filter for granularities

* dedup month in standardGranularitiesParents

* feat(schema-compiler): update cube's schema to allow granularities for time dimensions

* fix custom granularity processing in time dimension

* update yaml schema compiler to process custom granularities

* Add custom granularities support to time dimension and base query

* feat(schema-compiler): update cube's schema to allow granularities for time dimensions (simple syntax)

* Add custom granularities simple syntax support to the time dimension and base query

* Rename baseGranularity → rollupGranularity

* fix schema validation

* fix custom granularity processing in BaseTimeDimension

* Moved custom granularity processing from time dimension to base query

* Revert: deep evaluateSymbolSql processing

* Implement dimensionTimeGroupedColumn for Snowflake

* Implement dimensionTimeGroupedColumn for MySQL

* Add custom granularity time intervals generation (used in rollups)

* Rewrite parseSqlInterval from regex to split

* Add tests for timeSeriesFromCustomInterval

* Add tests for yaml schema compiler

* Add tests for cube validator

* Fix overTimeSeriesQuery with custom granularities

* Add tests for sql query generation for queries with custom granularities

* return back deleted test

* Remove comment, add types

* update cube validation scheme: remove sql, add origin

* add tests for cube validator

* add another test for yaml schema compiler

* move isGranularityNaturalAligned() to utils to be reused

* Add Granularity entity and move all related processing there

also implement date_bin for postgres

* temporary comment out dimensionTimeGroupedColumn

* Fix timeSeriesFromCustomInterval generation

* Fix tests timeSeriesFromCustomInterval

* Add more tests for timeSeriesFromCustomInterval

* Fix Granularity constructor

* Add integration tests for Custom Granularities for PostgreSQL

* Fix Custom Granularities tests in CI (needs order by)

* Implement date_bin for Snowflake

* Move BaseDbRunner from postgres to separate folder

* fix customGranularity.origin init

* implement dateBin in MySQL

* fix interval math ops in MySQL

* fix postgres custom granularities tests

* Add tests for custom granularities in MySQL

* move granularityFromIntervalString from BaseQuery → Granularity

* fix granularityFromIntervalString

* improve addInterval / subtractInterval in MS SQL Query (now supports intervals with >1 time units)

* Implement dateBin for MS SQL

* Add tests for custom granularities in MS SQL

* fix 2 granularities test (actually revert the bad changes back)

* add comment for dateBin in postgresql query

* Implement dateBin for Databricks

* Implement dateBin for DuckDB

* implement dateBin for BigQuery

* extend testing fixtures with custom granularities

* fix TimeDimensionGranularity type to allow custom granularities

* Add more unit tests for granularities

* fix writing yaml with test cubes definitions containing origin with dashes

* add tests for custom granularities in testing drivers

* skip custom granularities tests for athena

* skip custom granularities tests for clickhouse

* add test snapshots for custom granularities tests for postgres

* skip custom granularities tests for mysql (some)

* add test snapshots for custom granularities tests for ms sql

* skip custom granularities tests for mssql (some)

* add test snapshots for custom granularities tests for snowflake

* add test snapshots for custom granularities tests for bigQuery

* skip custom granularities tests for databricks (some)

* add test snapshots for custom granularities tests for Databricks

* Implement dateBin for ClickHouse

* Add custom granularities integration tests for ClickHouse

* unskip custom granularities tests for clickhouse (some)

* add test snapshots for custom granularities tests for ClickHouse

* fix linting

* fix integration tests for clickhouse v22.x

* fix in dateBin for Postgresql and Bigquery

* Add char limits for granularities names in queries

* Add time series date range limit checking before generating

* add tests for limit checking in time series generation

* Improve error message for time series limit

* fix tests for time series limit

* Refactor: move isGranularityNaturalAligned from Basequery → Granularity class

* add test snapshots for custom granularities tests for MySQL

* move diffTimeUnitForInterval to BaseQuery

* fix in choosing minGranularity for granularity with origin point

Author: KSDaemon

packages/cubejs-api-gateway/src/gateway.ts

@@ -1231,7 +1231,7 @@ class ApiGateway {
       if (queryGranularity.length > 1) {
         throw new UserError('Data blending query granularities must match');
       }
-      if (queryGranularity.filter(Boolean).length === 0) {
+      if (queryGranularity.length === 0) {
         throw new UserError('Data blending query without granularity is not supported');
       }
     }

packages/cubejs-api-gateway/src/query.js

@@ -103,7 +103,7 @@ const querySchema = Joi.object().keys({
   filters: Joi.array().items(oneFilter, oneCondition),
   timeDimensions: Joi.array().items(Joi.object().keys({
     dimension: id.required(),
-    granularity: Joi.valid('quarter', 'day', 'month', 'year', 'week', 'hour', 'minute', 'second', null),
+    granularity: Joi.string().max(128, 'utf8'), // Custom granularities may have arbitrary names
     dateRange: [
       Joi.array().items(Joi.string()).min(1).max(2),
       Joi.string()

packages/cubejs-backend-shared/src/time.ts

@@ -5,7 +5,12 @@ const Moment = require('moment-timezone');
 
 const moment = extendMoment(Moment);
 
-type QueryDateRange = [string, string];
+export type QueryDateRange = [string, string];
+type SqlInterval = string;
+export type TimeSeriesOptions = {
+  timestampPrecision: number
+};
+type ParsedInterval = Partial<Record<unitOfTime.DurationConstructor, number>>;
 
 export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision: number) => QueryDateRange[]> = {
   day: (range: DateRange, digits) => Array.from(range.snapTo('day').by('day'))
@@ -26,26 +31,159 @@ export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision:
     .map(d => [d.format(`YYYY-MM-DDT00:00:00.${'0'.repeat(digits)}`), d.endOf('quarter').format(`YYYY-MM-DDT23:59:59.${'9'.repeat(digits)}`)]),
 };
 
-type TimeSeriesOptions = {
-  timestampPrecision: number
+/**
+ * Parse PostgreSQL-like interval string into object
+ * E.g. '2 years 15 months 100 weeks 99 hours 15 seconds'
+ * Negative units are also supported
+ * E.g. '-2 months 5 days -10 hours'
+ */
+export function parseSqlInterval(intervalStr: SqlInterval): ParsedInterval {
+  const interval: ParsedInterval = {};
+  const parts = intervalStr.split(/\s+/);
+
+  for (let i = 0; i < parts.length; i += 2) {
+    const value = parseInt(parts[i], 10);
+    const unit = parts[i + 1];
+
+    // Remove ending 's' (e.g., 'days' -> 'day')
+    const singularUnit = (unit.endsWith('s') ? unit.slice(0, -1) : unit) as unitOfTime.DurationConstructor;
+    interval[singularUnit] = value;
+  }
+
+  return interval;
+}
+
+export function addInterval(date: moment.Moment, interval: ParsedInterval): moment.Moment {
+  const res = date.clone();
+
+  Object.entries(interval).forEach(([key, value]) => {
+    res.add(value, key as unitOfTime.DurationConstructor);
+  });
+
+  return res;
+}
+
+export function subtractInterval(date: moment.Moment, interval: ParsedInterval): moment.Moment {
+  const res = date.clone();
+
+  Object.entries(interval).forEach(([key, value]) => {
+    res.subtract(value, key as unitOfTime.DurationConstructor);
+  });
+
+  return res;
+}
+
+/**
+ * Returns the closest date prior to date parameter aligned with the origin point
+ */
+function alignToOrigin(startDate: moment.Moment, interval: ParsedInterval, origin: moment.Moment): moment.Moment {
+  let alignedDate = startDate.clone();
+  let intervalOp;
+  let isIntervalNegative = false;
+
+  let offsetDate = addInterval(origin, interval);
+
+  // The easiest way to check the interval sign
+  if (offsetDate.isBefore(origin)) {
+    isIntervalNegative = true;
+  }
+
+  offsetDate = origin.clone();
+
+  if (startDate.isBefore(origin)) {
+    intervalOp = isIntervalNegative ? addInterval : subtractInterval;
+
+    while (offsetDate.isAfter(startDate)) {
+      offsetDate = intervalOp(offsetDate, interval);
+    }
+    alignedDate = offsetDate;
+  } else {
+    intervalOp = isIntervalNegative ? subtractInterval : addInterval;
+
+    while (offsetDate.isBefore(startDate)) {
+      alignedDate = offsetDate.clone();
+      offsetDate = intervalOp(offsetDate, interval);
+    }
+
+    if (offsetDate.isSame(startDate)) {
+      alignedDate = offsetDate;
+    }
+  }
+
+  return alignedDate;
+}
+
+function parsedSqlIntervalToDuration(parsedInterval: ParsedInterval): moment.Duration {
+  const duration = moment.duration();
+
+  Object.entries(parsedInterval).forEach(([key, value]) => {
+    duration.add(value, key as unitOfTime.DurationConstructor);
+  });
+
+  return duration;
+}
+
+function checkSeriesForDateRange(intervalStr: string, [startStr, endStr]: QueryDateRange): void {
+  const intervalParsed = parseSqlInterval(intervalStr);
+  const intervalAsSeconds = parsedSqlIntervalToDuration(intervalParsed).asSeconds();
+  const start = moment(startStr);
+  const end = moment(endStr);
+  const rangeSeconds = end.diff(start, 'seconds');
+
+  const limit = 50000; // TODO Make this as configurable soft limit
+  const count = rangeSeconds / intervalAsSeconds;
+
+  if (count > limit) {
+    throw new Error(`The count of generated date ranges (${count}) for the request from [${startStr}] to [${endStr}] by ${intervalStr} is over limit (${limit}). Please reduce the requested date interval or use bigger granularity.`);
+  }
+}
+
+export const timeSeriesFromCustomInterval = (intervalStr: string, [startStr, endStr]: QueryDateRange, origin: moment.Moment, options: TimeSeriesOptions = { timestampPrecision: 3 }): QueryDateRange[] => {
+  checkSeriesForDateRange(intervalStr, [startStr, endStr]);
+
+  const intervalParsed = parseSqlInterval(intervalStr);
+  const start = moment(startStr);
+  const end = moment(endStr);
+  let alignedStart = alignToOrigin(start, intervalParsed, origin);
+
+  const dates: QueryDateRange[] = [];
+
+  while (alignedStart.isBefore(end)) {
+    const s = alignedStart.clone();
+    alignedStart = addInterval(alignedStart, intervalParsed);
+    dates.push([
+      s.format(`YYYY-MM-DDTHH:mm:ss.${'0'.repeat(options.timestampPrecision)}`),
+      alignedStart.clone()
+        .subtract(1, 'second')
+        .format(`YYYY-MM-DDTHH:mm:ss.${'9'.repeat(options.timestampPrecision)}`)
+    ]);
+  }
+
+  return dates;
 };
 
+/**
+ * Returns array of date ranges for a predefined granularity aligned with the start of the year as pivot point
+ */
 export const timeSeries = (granularity: string, dateRange: QueryDateRange, options: TimeSeriesOptions = { timestampPrecision: 3 }): QueryDateRange[] => {
   if (!TIME_SERIES[granularity]) {
-    // TODO error
     throw new Error(`Unsupported time granularity: ${granularity}`);
   }
 
   if (!options.timestampPrecision) {
     throw new Error(`options.timestampPrecision is required, actual: ${options.timestampPrecision}`);
   }
 
+  checkSeriesForDateRange(`1 ${granularity}`, dateRange);
+
   // moment.range works with strings
   const range = moment.range(<any>dateRange[0], <any>dateRange[1]);
 
   return TIME_SERIES[granularity](range, options.timestampPrecision);
 };
 
+export const isPredefinedGranularity = (granularity: string): boolean => !!TIME_SERIES[granularity];
+
 export const FROM_PARTITION_RANGE = '__FROM_PARTITION_RANGE';
 
 export const TO_PARTITION_RANGE = '__TO_PARTITION_RANGE';

packages/cubejs-backend-shared/test/time.test.ts

@@ -1,4 +1,5 @@
-import { inDbTimeZone, timeSeries } from '../src';
+import moment from 'moment-timezone';
+import { inDbTimeZone, timeSeries, isPredefinedGranularity, timeSeriesFromCustomInterval } from '../src';
 
 describe('time', () => {
   it('time series - day', () => {
@@ -29,6 +30,154 @@ describe('time', () => {
     ]);
   });
 
+  it('time series - reach limits', () => {
+    expect(() => {
+      timeSeries('second', ['1970-01-01', '2021-01-02']);
+    }).toThrowError(/The count of generated date ranges.*for the request.*is over limit/);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2021-01-01', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2023-12-31'], moment('2021-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-12-31T23:59:59.999'],
+      ['2022-01-01T00:00:00.000', '2022-12-31T23:59:59.999'],
+      ['2023-01-01T00:00:00.000', '2023-12-31T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2020-01-01', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2023-12-31'], moment('2020-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-12-31T23:59:59.999'],
+      ['2022-01-01T00:00:00.000', '2022-12-31T23:59:59.999'],
+      ['2023-01-01T00:00:00.000', '2023-12-31T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2025-01-01', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2023-12-31'], moment('2025-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-12-31T23:59:59.999'],
+      ['2022-01-01T00:00:00.000', '2022-12-31T23:59:59.999'],
+      ['2023-01-01T00:00:00.000', '2023-12-31T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2025-03-01', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2022-12-31'], moment('2025-03-01'))).toEqual([
+      ['2020-03-01T00:00:00.000', '2021-02-28T23:59:59.999'],
+      ['2021-03-01T00:00:00.000', '2022-02-28T23:59:59.999'],
+      ['2022-03-01T00:00:00.000', '2023-02-28T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2015-03-01', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2022-12-31'], moment('2015-03-01'))).toEqual([
+      ['2020-03-01T00:00:00.000', '2021-02-28T23:59:59.999'],
+      ['2021-03-01T00:00:00.000', '2022-02-28T23:59:59.999'],
+      ['2022-03-01T00:00:00.000', '2023-02-28T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2020-03-15', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2022-12-31'], moment('2020-03-15'))).toEqual([
+      ['2020-03-15T00:00:00.000', '2021-03-14T23:59:59.999'],
+      ['2021-03-15T00:00:00.000', '2022-03-14T23:59:59.999'],
+      ['2022-03-15T00:00:00.000', '2023-03-14T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 year, origin - 2019-03-15', () => {
+    expect(timeSeriesFromCustomInterval('1 year', ['2021-01-01', '2022-12-31'], moment('2019-03-15'))).toEqual([
+      ['2020-03-15T00:00:00.000', '2021-03-14T23:59:59.999'],
+      ['2021-03-15T00:00:00.000', '2022-03-14T23:59:59.999'],
+      ['2022-03-15T00:00:00.000', '2023-03-14T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 2 months, origin - 2019-01-01', () => {
+    expect(timeSeriesFromCustomInterval('2 months', ['2021-01-01', '2021-12-31'], moment('2019-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-02-28T23:59:59.999'],
+      ['2021-03-01T00:00:00.000', '2021-04-30T23:59:59.999'],
+      ['2021-05-01T00:00:00.000', '2021-06-30T23:59:59.999'],
+      ['2021-07-01T00:00:00.000', '2021-08-31T23:59:59.999'],
+      ['2021-09-01T00:00:00.000', '2021-10-31T23:59:59.999'],
+      ['2021-11-01T00:00:00.000', '2021-12-31T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 2 months, origin - 2019-03-15', () => {
+    expect(timeSeriesFromCustomInterval('2 months', ['2021-01-01', '2021-12-31'], moment('2019-03-15'))).toEqual([
+      ['2020-11-15T00:00:00.000', '2021-01-14T23:59:59.999'],
+      ['2021-01-15T00:00:00.000', '2021-03-14T23:59:59.999'],
+      ['2021-03-15T00:00:00.000', '2021-05-14T23:59:59.999'],
+      ['2021-05-15T00:00:00.000', '2021-07-14T23:59:59.999'],
+      ['2021-07-15T00:00:00.000', '2021-09-14T23:59:59.999'],
+      ['2021-09-15T00:00:00.000', '2021-11-14T23:59:59.999'],
+      ['2021-11-15T00:00:00.000', '2022-01-14T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 1 months 2 weeks 3 days, origin - 2021-01-25', () => {
+    expect(timeSeriesFromCustomInterval('1 months 2 weeks 3 days', ['2021-01-01', '2021-12-31'], moment('2021-01-25'))).toEqual([
+      ['2020-12-08T00:00:00.000', '2021-01-24T23:59:59.999'],
+      ['2021-01-25T00:00:00.000', '2021-03-13T23:59:59.999'],
+      ['2021-03-14T00:00:00.000', '2021-04-30T23:59:59.999'],
+      ['2021-05-01T00:00:00.000', '2021-06-17T23:59:59.999'],
+      ['2021-06-18T00:00:00.000', '2021-08-03T23:59:59.999'],
+      ['2021-08-04T00:00:00.000', '2021-09-20T23:59:59.999'],
+      ['2021-09-21T00:00:00.000', '2021-11-06T23:59:59.999'],
+      ['2021-11-07T00:00:00.000', '2021-12-23T23:59:59.999'],
+      ['2021-12-24T00:00:00.000', '2022-02-09T23:59:59.999'],
+    ]);
+  });
+
+  it('time series - custom: interval - 3 weeks, origin - 2020-12-15', () => {
+    expect(timeSeriesFromCustomInterval('3 weeks', ['2021-01-01', '2021-03-01'], moment('2020-12-15'))).toEqual([
+      ['2020-12-15T00:00:00.000', '2021-01-04T23:59:59.999'],
+      ['2021-01-05T00:00:00.000', '2021-01-25T23:59:59.999'],
+      ['2021-01-26T00:00:00.000', '2021-02-15T23:59:59.999'],
+      ['2021-02-16T00:00:00.000', '2021-03-08T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 6 months, origin - 2021-01-01', () => {
+    expect(timeSeriesFromCustomInterval('6 months', ['2021-01-01', '2021-12-31'], moment('2021-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-06-30T23:59:59.999'],
+      ['2021-07-01T00:00:00.000', '2021-12-31T23:59:59.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 2 months 3 weeks 4 days 5 hours 6 minutes 7 seconds, origin - 2021-01-01', () => {
+    expect(timeSeriesFromCustomInterval('2 months 3 weeks 4 days 5 hours 6 minutes 7 seconds', ['2021-01-01', '2021-12-31'], moment('2021-01-01'))).toEqual([
+      ['2021-01-01T00:00:00.000', '2021-03-26T05:06:06.999'],
+      ['2021-03-26T05:06:07.000', '2021-06-20T10:12:13.999'],
+      ['2021-06-20T10:12:14.000', '2021-09-14T15:18:20.999'],
+      ['2021-09-14T15:18:21.000', '2021-12-09T20:24:27.999'],
+      ['2021-12-09T20:24:28.000', '2022-03-07T01:30:34.999']
+    ]);
+  });
+
+  it('time series - custom: interval - 10 minutes 15 seconds, origin - 2021-02-01 09:59:45', () => {
+    expect(timeSeriesFromCustomInterval('10 minutes 15 seconds', ['2021-02-01 10:00:00', '2021-02-01 12:00:00'], moment('2021-02-01 09:59:45'))).toEqual([
+      ['2021-02-01T09:59:45.000', '2021-02-01T10:09:59.999'],
+      ['2021-02-01T10:10:00.000', '2021-02-01T10:20:14.999'],
+      ['2021-02-01T10:20:15.000', '2021-02-01T10:30:29.999'],
+      ['2021-02-01T10:30:30.000', '2021-02-01T10:40:44.999'],
+      ['2021-02-01T10:40:45.000', '2021-02-01T10:50:59.999'],
+      ['2021-02-01T10:51:00.000', '2021-02-01T11:01:14.999'],
+      ['2021-02-01T11:01:15.000', '2021-02-01T11:11:29.999'],
+      ['2021-02-01T11:11:30.000', '2021-02-01T11:21:44.999'],
+      ['2021-02-01T11:21:45.000', '2021-02-01T11:31:59.999'],
+      ['2021-02-01T11:32:00.000', '2021-02-01T11:42:14.999'],
+      ['2021-02-01T11:42:15.000', '2021-02-01T11:52:29.999'],
+      ['2021-02-01T11:52:30.000', '2021-02-01T12:02:44.999']
+    ]);
+  });
+
+  it('time series - custom: interval - reach limits', () => {
+    expect(() => {
+      timeSeriesFromCustomInterval('10 minutes 15 seconds', ['1970-01-01', '2021-01-02'], moment('2021-02-01 09:59:45'));
+    }).toThrowError(/The count of generated date ranges.*for the request.*is over limit/);
+  });
+
   it('inDbTimeZone', () => {
     expect(inDbTimeZone('UTC', 'YYYY-MM-DD[T]HH:mm:ss.SSSSSS[Z]', '2020-01-01T00:00:00.000000')).toEqual(
       '2020-01-01T00:00:00.000000Z'
@@ -38,4 +187,9 @@ describe('time', () => {
       '2020-01-31T23:59:59.999999Z'
     );
   });
+
+  it('isPredefinedGranularity', () => {
+    expect(isPredefinedGranularity('day')).toBeTruthy();
+    expect(isPredefinedGranularity('fiscal_year_by_1st_feb')).toBeFalsy();
+  });
 });

packages/cubejs-client-core/index.d.ts

@@ -787,7 +787,9 @@ declare module '@cubejs-client/core' {
     | 'afterDate'
     | 'afterOrOnDate';
 
-  export type TimeDimensionGranularity = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year';
+  export type TimeDimensionPredefinedGranularity = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year';
+
+  export type TimeDimensionGranularity = TimeDimensionPredefinedGranularity | string;
 
   export type DateRange = string | [string, string];