Skip to content
This repository has been archived by the owner on Dec 11, 2022. It is now read-only.

Commit

Permalink
Handle time range filtering issue for both partitioned and non-partit…
Browse files Browse the repository at this point in the history 
…ioned (#385)

* Handle time range filtering issue for both partitioned and non-partitioned (fixes #321, #325, #334, #362)

* Fix UTC conversion & usage
  • Loading branch information
@ofir5300
ofir5300 authored Nov 21, 2021
1 parent 370b61a commit b5f2daf
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 49 deletions.
73 changes: 43 additions & 30 deletions src/bigquery_query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,12 @@ export default class BigQueryQuery {
return res;
}

public static formatDateToString(inputDate, separator = '', addtime = false) {
const date = new Date(inputDate);
public static formatDateToString(inputDate, convertToUTC = false, separator = '', addtime = false) {
let date = new Date(inputDate);
if (convertToUTC) {
// check if should convert string to UTC time (relevant whenever addTime = true)
date = BigQueryQuery.convertToUtc(date);
}
// 01, 02, 03, ... 29, 30, 31
const DD = (date.getDate() < 10 ? '0' : '') + date.getDate();
// 01, 02, 03, ... 10, 11, 12
Expand Down Expand Up @@ -89,6 +93,8 @@ export default class BigQueryQuery {
public static replaceTimeShift(q) {
return q.replace(/(\$__timeShifting\().*?(?=\))./g, '');
}

// Note: converts to UTC time BUT - Date object always represented in local time (so probably relevant when handling date/time strings)
static convertToUtc(d) {
return new Date(d.getTime() + d.getTimezoneOffset() * 60000);
}
Expand Down Expand Up @@ -378,22 +384,32 @@ export default class BigQueryQuery {
return query;
}

// Detects either date or timestamp, only if not comment out
static hasDateFilter(whereClause) {
return whereClause.match(/((?<!--.*)([1-2]\d{3}-[0-1]\d-[0-3]\d)|([\D](\d{13})[\D]).*\n)/gi);
}

public buildWhereClause() {
let query = '';
let query = '', hasMacro = false, hasDateFilter = false;
const conditions = _.map(this.target.where, (tag, index) => {
switch (tag.type) {
case 'macro':
hasMacro = true;
return tag.name + '(' + this.target.timeColumn + ')';
case 'expression':
return tag.params.join(' ');
const expression = tag.params.join(' ');
hasDateFilter = BigQueryQuery.hasDateFilter(expression) ? true : hasDateFilter;
return expression;
}
});
const hasTimeFilter = !!(hasMacro || hasDateFilter);
if (this.target.partitioned) {
const partitionedField = this.target.partitionedField ? this.target.partitionedField : '_PARTITIONTIME';
if (this.target.timeColumn !== partitionedField) {
if (this.target.timeColumn !== partitionedField && !hasTimeFilter) {
if (this.templateSrv.timeRange && this.templateSrv.timeRange.from) {
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(
this.templateSrv.timeRange.from._d,
this.target.convertToUTC,
'-',
true
)}'`;
Expand All @@ -402,6 +418,7 @@ export default class BigQueryQuery {
if (this.templateSrv.timeRange && this.templateSrv.timeRange.to) {
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(
this.templateSrv.timeRange.to._d,
this.target.convertToUTC,
'-',
true
)}'`;
Expand All @@ -410,8 +427,8 @@ export default class BigQueryQuery {
}
}
if (this.target.sharded) {
const from = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d);
const to = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d);
const from = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d, this.target.convertToUTC);
const to = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d, this.target.convertToUTC);
const sharded = "_TABLE_SUFFIX BETWEEN '" + from + "' AND '" + to + "' ";
conditions.push(sharded);
}
Expand Down Expand Up @@ -536,24 +553,18 @@ export default class BigQueryQuery {
public expend_macros(options) {
if (this.target.rawSql) {
let q = this.target.rawSql;
q = BigQueryQuery.replaceTimeShift(q);
q = this.replaceTimeFilters(q, options);
q = this.replacetimeGroupAlias(q, true, options);
q = this.replacetimeGroupAlias(q, false, options);
return q;
let hasTimeFilter, hasTimeGroup, hasTimeGroupAlias = false;
q = BigQueryQuery.replaceTimeShift(q); // should also block additional partition time filtering?
[q, hasTimeFilter] = this.replaceTimeFilters(q, options);
[q, hasTimeGroup] = this.replacetimeGroupAlias(q, true, options);
[q, hasTimeGroupAlias] = this.replacetimeGroupAlias(q, false, options);
return [q, hasTimeFilter || hasTimeGroup || hasTimeGroupAlias, this.target.convertToUTC];
}
}
public replaceTimeFilters(q, options) {
let fromD = options.range.from;
let toD = options.range.to;
if (this.target.convertToUTC === true) {
fromD = BigQueryQuery.convertToUtc(options.range.from._d);
toD = BigQueryQuery.convertToUtc(options.range.to._d);
}
let to = '';
let from = '';
from = this._getDateRangePart(fromD);
to = this._getDateRangePart(toD);
const { from: fromD, to: toD } = options.range;
const from = this._getDateRangePart(fromD, this.target.convertToUTC);
const to = this._getDateRangePart(toD, this.target.convertToUTC);
if (this.target.timeColumn === '-- time --') {
const myRegexp = /\$__timeFilter\(([\w_.]+)\)/g;
const tf = myRegexp.exec(q);
Expand All @@ -564,26 +575,27 @@ export default class BigQueryQuery {
const range = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' BETWEEN ' + from + ' AND ' + to;
const fromRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' > ' + from + ' ';
const toRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' < ' + to + ' ';
const hasMacro = q.match(/(\b__timeFilter\b)|(\b__timeFrom\b)|(\b__timeTo\b)|(\b__millisTimeTo\b)|(\b__millisTimeFrom\b)/g)
q = q.replace(/\$__timeFilter\((.*?)\)/g, range);
q = q.replace(/\$__timeFrom\(([\w_.]+)\)/g, fromRange);
q = q.replace(/\$__timeTo\(([\w_.]+)\)/g, toRange);
q = q.replace(/\$__millisTimeTo\(([\w_.]+)\)/g, to);
q = q.replace(/\$__millisTimeFrom\(([\w_.]+)\)/g, from);
return q;
return [q, hasMacro];
}

public replacetimeGroupAlias(q, alias: boolean, options) {
const res = BigQueryQuery._getInterval(q, alias);
const interval = res[0];
const mininterval = res[1];
if (!interval) {
return q;
return [q, false];
}
const intervalStr = this.getIntervalStr(interval, mininterval, options);
if (alias) {
return q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
return [q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroupAlias\b)/g)];
} else {
return q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
return [q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroup\b)/g)];
}
}

Expand All @@ -597,13 +609,14 @@ export default class BigQueryQuery {
const seconds = (this.templateSrv.timeRange.to._d - this.templateSrv.timeRange.from._d) / 1000;
return Math.ceil(seconds / options.maxDataPoints) + 's';
}
private _getDateRangePart(part) {
private _getDateRangePart(part, convertToUTC = false) {
// if its TIMESTAMP no need conversion (same value for utc or local), else - need conversion
if (this.target.timeColumnType === 'DATE') {
return "'" + BigQueryQuery.formatDateToString(part, '-') + "'";
return "'" + BigQueryQuery.formatDateToString(part, convertToUTC, '-') + "'"; // "'2021-01-31'"
} else if (this.target.timeColumnType === 'DATETIME') {
return "'" + BigQueryQuery.formatDateToString(part, '-', true) + "'";
return "'" + BigQueryQuery.formatDateToString(part, convertToUTC, '-', true) + "'"; // "'2021-01-31 19:41:45'"
} else {
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')';
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')'; // "TIMESTAMP_MILLIS (1612056873199)"
}
}
}
38 changes: 19 additions & 19 deletions src/datasource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ export class BigQueryDatasource {
return copy;
}

private static _updatePartition(q, options) {
private static _updatePartition(q, options, convertToUTC = false) {
if (q.indexOf('AND _PARTITIONTIME >= ') < 1) {
return q;
}
Expand All @@ -137,26 +137,26 @@ export class BigQueryDatasource {
}
const from = q.substr(q.indexOf('AND _PARTITIONTIME >= ') + 22, 21);

const newFrom = "'" + BigQueryQuery.formatDateToString(options.range.from._d, '-', true) + "'";
const newFrom = "'" + BigQueryQuery.formatDateToString(options.range.from._d, convertToUTC, '-', true) + "'";
q = q.replace(from, newFrom);
const to = q.substr(q.indexOf('AND _PARTITIONTIME < ') + 21, 21);
const newTo = "'" + BigQueryQuery.formatDateToString(options.range.to._d, '-', true) + "'";
const newTo = "'" + BigQueryQuery.formatDateToString(options.range.to._d, convertToUTC, '-', true) + "'";

q = q.replace(to, newTo) + '\n ';
return q;
}

private static _updateTableSuffix(q, options) {
private static _updateTableSuffix(q, options, convertToUTC = false) {
const ind = q.indexOf('AND _TABLE_SUFFIX BETWEEN ');
if (ind < 1) {
return q;
}
const from = q.substr(ind + 28, 8);

const newFrom = BigQueryQuery.formatDateToString(options.range.from._d);
const newFrom = BigQueryQuery.formatDateToString(options.range.from._d, convertToUTC);
q = q.replace(from, newFrom);
const to = q.substr(ind + 43, 8);
const newTo = BigQueryQuery.formatDateToString(options.range.to._d);
const newTo = BigQueryQuery.formatDateToString(options.range.to._d, convertToUTC);
q = q.replace(to, newTo) + '\n ';
return q;
}
Expand Down Expand Up @@ -270,7 +270,7 @@ export class BigQueryDatasource {
});
this.queryModel.target.rawSql = query.rawSql;
modOptions = BigQueryDatasource._setupTimeShiftQuery(query, options);
const q = this.setUpQ(modOptions, options, query);
const q = this.setUpQ(modOptions, options, query); // TODO hanle raw sql WHERE clause!
console.log(q);
return this.doQuery(q, options.panelId + query.refId, query.queryPriority).then((response) => {
return ResponseParser.parseDataQuery(response, query.format);
Expand Down Expand Up @@ -426,7 +426,7 @@ export class BigQueryDatasource {
refId: options.annotation.name,
};
this.queryModel.target.rawSql = query.rawSql;
query.rawSql = this.queryModel.expend_macros(options);
[query.rawSql,] = this.queryModel.expend_macros(options);
return this.backendSrv
.datasourceRequest({
data: {
Expand All @@ -444,11 +444,11 @@ export class BigQueryDatasource {
.then((data) => this.responseParser.transformAnnotationResponse(options, data));
}
private setUpQ(modOptions, options, query) {
let q = this.queryModel.expend_macros(modOptions);
let [q, hasMacro, convertToUTC] = this.queryModel.expend_macros(modOptions);
if (q) {
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions);
q = BigQueryDatasource._updatePartition(q, modOptions);
q = BigQueryDatasource._updateTableSuffix(q, modOptions);
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions, hasMacro, convertToUTC);
q = BigQueryDatasource._updatePartition(q, modOptions, convertToUTC);
q = BigQueryDatasource._updateTableSuffix(q, modOptions, convertToUTC);
if (query.refId.search(Shifted) > -1) {
q = this._updateAlias(q, modOptions, query.refId);
}
Expand All @@ -466,19 +466,19 @@ export class BigQueryDatasource {
return q;
}
/**
* Add partition to query unless it has one
* Add partition to query unless it has one OR already being ranged by other condition
* @param query
* @param isPartitioned
* @param partitionedField
* @param options
*/
private setUpPartition(query, isPartitioned, partitionedField, options) {
private setUpPartition(query, isPartitioned, partitionedField, options, hasMacro = false, convertToUTC = false) {
partitionedField = partitionedField ? partitionedField : '_PARTITIONTIME';
if (isPartitioned && !query.match(new RegExp(partitionedField, "i"))) {
const fromD = BigQueryQuery.convertToUtc(options.range.from._d);
const toD = BigQueryQuery.convertToUtc(options.range.to._d);
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, '-', true)}'`;
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(toD, '-', true)}'`;
const hasTimeFilter = !!(BigQueryQuery.hasDateFilter(query.split(/where/gi)[1] || "") || hasMacro);
if (isPartitioned && !hasTimeFilter) {
const { from: { _d: fromD }, to: { _d: toD } } = options.range;
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, convertToUTC, '-', true)}'`;
const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(toD, convertToUTC, '-', true)}'`;
const partition = `where ${from} AND ${to} AND `;
if (query.match(/where/i)) query = query.replace(/where/i, partition);
else {
Expand Down

0 comments on commit b5f2daf

Please sign in to comment.