From db10c1bebc09de27d3100a9ef208f29328b9ee41 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 28 Apr 2021 18:02:18 +0200 Subject: [PATCH 01/31] make basic time offset work --- .../data/common/search/aggs/agg_config.ts | 8 + .../data/common/search/aggs/agg_configs.ts | 230 ++++++++++++++---- .../data/common/search/aggs/metrics/avg_fn.ts | 7 + .../data/common/search/aggs/metrics/count.ts | 7 +- .../common/search/aggs/metrics/count_fn.ts | 7 + .../data/common/search/aggs/metrics/max_fn.ts | 7 + .../common/search/aggs/metrics/median_fn.ts | 7 + .../search/aggs/metrics/metric_agg_type.ts | 10 +- .../data/common/search/aggs/metrics/min_fn.ts | 7 + .../aggs/metrics/single_percentile_fn.ts | 7 + .../data/common/search/aggs/metrics/sum_fn.ts | 7 + .../common/search/aggs/metrics/top_hit_fn.ts | 7 + src/plugins/data/common/search/aggs/types.ts | 1 + .../expressions/esaggs/request_handler.ts | 68 +++++- .../search/search_source/search_source.ts | 33 ++- .../data/common/search/tabify/tabify.ts | 2 +- .../editor_frame/frame_layout.tsx | 2 +- .../indexpattern_datasource/to_expression.ts | 3 + 18 files changed, 355 insertions(+), 65 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 283d276a22904..42dc52c59d822 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import moment from 'moment'; import _ from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign, Ensure } from '@kbn/utility-types'; @@ -172,6 +173,13 @@ export class AggConfig { return _.get(this.params, key); } + getTimeShift(): undefined | moment.Duration { + const rawTimeShift = this.getParam('timeShift'); + if (!rawTimeShift) return undefined; + const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); + return moment.duration(Number(amount), unit); + } + write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 2932ef7325aed..eeb370765a77b 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -6,11 +6,13 @@ * Side Public License, v 1. */ -import _ from 'lodash'; +import moment from 'moment'; +import _, { cloneDeep, isArray } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; +import { Aggregate, FiltersAggregate, FiltersBucketItem } from '@elastic/elasticsearch/api/types'; -import { ISearchOptions, ISearchSource } from 'src/plugins/data/public'; +import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; import { IAggType } from './agg_type'; import { AggTypesRegistryStart } from './agg_types_registry'; @@ -183,6 +185,9 @@ export class AggConfigs { let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; + const timeShifts = this.getTimeShifts(); + const hasTimeShifts = Object.keys(timeShifts).length > 0; + if (this.hierarchical) { // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs @@ -196,57 +201,104 @@ export class AggConfigs { }; }); } - this.getRequestAggs() - .filter((config: AggConfig) => !config.type.hasNoDsl) - .forEach((config: AggConfig, i: number, list) => { - if (!dslLvlCursor) { - // start at the top level - dslLvlCursor = dslTopLvl; - } else { - const prevConfig: AggConfig = list[i - 1]; - const prevDsl = dslLvlCursor[prevConfig.id]; + this.getRequestAggs().forEach((config: AggConfig, i: number, list) => { + if (!dslLvlCursor) { + // start at the top level + dslLvlCursor = dslTopLvl; + } else { + const prevConfig: AggConfig = list[i - 1]; + const prevDsl = dslLvlCursor[prevConfig.id]; + + // advance the cursor and nest under the previous agg, or + // put it on the same level if the previous agg doesn't accept + // sub aggs + dslLvlCursor = prevDsl?.aggs || dslLvlCursor; + } - // advance the cursor and nest under the previous agg, or - // put it on the same level if the previous agg doesn't accept - // sub aggs - dslLvlCursor = prevDsl?.aggs || dslLvlCursor; - } + if (hasTimeShifts) { + dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); + } - const dsl = config.type.hasNoDslParams - ? config.toDsl(this) - : (dslLvlCursor[config.id] = config.toDsl(this)); - let subAggs: any; + if (config.type.hasNoDsl) { + return; + } - parseParentAggs(dslLvlCursor, dsl); + const dsl = config.type.hasNoDslParams + ? config.toDsl(this) + : (dslLvlCursor[config.id] = config.toDsl(this)); + let subAggs: any; - if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { - // buckets that are not the last item in the list accept sub-aggs - subAggs = dsl.aggs || (dsl.aggs = {}); - } + parseParentAggs(dslLvlCursor, dsl); - if (subAggs) { - _.each(subAggs, (agg) => { - parseParentAggs(subAggs, agg); - }); - } - if (subAggs && nestedMetrics) { - nestedMetrics.forEach((agg: any) => { - subAggs[agg.config.id] = agg.dsl; - // if a nested metric agg has parent aggs, we have to add them to every level of the tree - // to make sure "bucket_path" references in the nested metric agg itself are still working - if (agg.dsl.parentAggs) { - Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => { - subAggs[parentAggId] = parentAgg; - }); - } - }); - } - }); + if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { + // buckets that are not the last item in the list accept sub-aggs + subAggs = dsl.aggs || (dsl.aggs = {}); + } + + if (subAggs) { + _.each(subAggs, (agg) => { + parseParentAggs(subAggs, agg); + }); + } + if (subAggs && nestedMetrics) { + nestedMetrics.forEach((agg: any) => { + subAggs[agg.config.id] = agg.dsl; + // if a nested metric agg has parent aggs, we have to add them to every level of the tree + // to make sure "bucket_path" references in the nested metric agg itself are still working + if (agg.dsl.parentAggs) { + Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => { + subAggs[parentAggId] = parentAgg; + }); + } + }); + } + }); removeParentAggs(dslTopLvl); return dslTopLvl; } + private insertTimeShiftSplit( + config: AggConfig, + timeShifts: Record, + dslLvlCursor: Record + ) { + if (this.isTimeShiftSplitAgg(config)) { + const filters: Record = {}; + // TODO validate this + const timeField = this.timeFields![0]; + filters['0'] = { + range: { + [timeField]: { + // only works if there is a time range + gte: this.timeRange!.from, + lte: this.timeRange!.to, + }, + }, + }; + Object.entries(timeShifts).forEach(([key, shift]) => { + filters[key] = { + range: { + [timeField]: { + // only works if there is a time range + gte: moment(this.timeRange!.from).subtract(shift).toISOString(), + lte: moment(this.timeRange!.to).subtract(shift).toISOString(), + }, + }, + }; + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; + + dslLvlCursor = dslLvlCursor.time_offset_split.aggs; + } + return dslLvlCursor; + } + getAll() { return [...this.aggs]; } @@ -289,6 +341,98 @@ export class AggConfigs { ); } + getTimeShifts(): Record { + const timeShifts: Record = {}; + this.getAll() + .filter((agg) => agg.schema === 'metric') + .map((agg) => agg.getTimeShift()) + .forEach((timeShift) => { + if (timeShift) { + timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } + }); + return timeShifts; + } + + isTimeShiftSplitAgg(aggConfig: AggConfig) { + // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here + const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); + // the first date histogram or the first metric (if there are no date histograms) is the level to add the time shift filter split + return ( + (hasDateHistograms && aggConfig.type.name === 'date_histogram') || + this.byType('metrics')[0] === aggConfig + ); + } + + postFlightTransform(response: IEsSearchResponse) { + const timeShifts = this.getTimeShifts(); + if (Object.keys(timeShifts).length === 0) { + return response; + } + const transformedRawResponse = cloneDeep(response.rawResponse); + const aggCursor = transformedRawResponse.aggregations!; + + // TODO: Add a "context" param here which keeps track of at which level of the response we are + const mergeObject = (target: any, source: any, shiftKey: string) => { + Object.entries(source).forEach(([key, val]) => { + if (typeof val !== 'object') { + if (key === 'doc_count') { + target[`doc_count_${shiftKey}`] = val; + } else { + target[key] = val; + } + } else { + const agg = this.byId(key); + if (agg && agg.type.type === 'metrics') { + const timeShift = agg.getTimeShift(); + if (!timeShift || String(timeShift.asMilliseconds()) !== shiftKey) { + // this is a metric from another time shift, do not copy over + return; + } + } + if (!target[key]) { + target[key] = {}; + } + mergeObject(target[key], source[key], shiftKey); + } + }); + }; + const transformTimeShift = (cursor: Record): undefined => { + if (cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< + string, + FiltersBucketItem + >; + const subTree = timeShiftedBuckets['0']; + // TODO pull in metrics from other buckets + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeObject(subTree, timeShiftedBuckets[key], key); + }); + + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + return; + } + // recurse deeper into the response object + Object.keys(cursor).forEach((subAggId) => { + const subAgg = cursor[subAggId]; + if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { + return; + } + if (isArray(subAgg.buckets)) { + subAgg.buckets.forEach(transformTimeShift); + } else { + Object.values(subAgg.buckets).forEach(transformTimeShift); + } + }); + }; + transformTimeShift(aggCursor); + return { + ...response, + rawResponse: transformedRawResponse, + }; + } + getRequestAggById(id: string) { return this.aggs.find((agg: AggConfig) => agg.id === id); } diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts index 253013238d10e..6b329a8bcfa99 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts @@ -62,6 +62,13 @@ export const aggAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index 8a10d7edb3f83..fac1751290f70 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -31,7 +31,12 @@ export const getCountMetricAgg = () => }; }, getValue(agg, bucket) { - return bucket.doc_count; + const timeShift = agg.getTimeShift(); + if (!timeShift) { + return bucket.doc_count; + } else { + return bucket[`doc_count_${timeShift.asMilliseconds()}`]; + } }, isScalable() { return true; diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.ts index 40c87db57eedc..7ca49078ee901 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.ts @@ -54,6 +54,13 @@ export const aggCount = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.ts index 7eac992680737..f42312a37ecd6 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.ts @@ -62,6 +62,13 @@ export const aggMax = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.ts index 1c0afd81a63c4..9145e088d83a9 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.ts @@ -67,6 +67,13 @@ export const aggMedian = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 3ebb771413665..45bc571c37515 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -11,7 +11,7 @@ import { AggType, AggTypeConfig } from '../agg_type'; import { AggParamType } from '../param_types/agg'; import { AggConfig } from '../agg_config'; import { METRIC_TYPES } from './metric_agg_types'; -import { FieldTypes } from '../param_types'; +import { BaseParamType, FieldTypes } from '../param_types'; export interface IMetricAggConfig extends AggConfig { type: InstanceType; @@ -47,6 +47,14 @@ export class MetricAggType) { super(config); + this.params.push( + new BaseParamType({ + name: 'timeShift', + type: 'string', + write: () => {}, + }) as MetricAggParam + ); + this.getValue = config.getValue || ((agg, bucket) => { diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.ts index 6dfbac1ecb8b4..36b576081ac6e 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.ts @@ -62,6 +62,13 @@ export const aggMin = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts index e7ef22c6faeee..b11ed2da8cc76 100644 --- a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts @@ -74,6 +74,13 @@ export const aggSinglePercentile = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts index d0175e0c8fafe..6e795ada78669 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts @@ -62,6 +62,13 @@ export const aggSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts index 85b54f1695493..9204ca88846c2 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts @@ -94,6 +94,13 @@ export const aggTopHit = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index 675be2323b93e..c0eb0c6c241a9 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -132,6 +132,7 @@ export type AggsStart = Assign 0 ? timeFields : defaultTimeFields; + aggs.setTimeRange(timeRange as TimeRange); - aggs.setTimeFields(timeFields); + aggs.setTimeFields(allTimeFields); // For now we need to mirror the history of the passed search source, since // the request inspector wouldn't work otherwise. @@ -90,19 +97,60 @@ export const handleRequest = async ({ return aggs.onSearchRequestStart(paramSearchSource, options); }); - // If timeFields have been specified, use the specified ones, otherwise use primary time field of index - // pattern if it's available. - const defaultTimeField = indexPattern?.getTimeField?.(); - const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : []; - const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; - // If a timeRange has been specified and we had at least one timeField available, create range // filters for that those time fields if (timeRange && allTimeFields.length > 0) { timeFilterSearchSource.setField('filter', () => { - return allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter); + const timeShifts = aggs.getTimeShifts(); + const hasTimeShift = Object.values(aggs.getTimeShifts()).length > 0; + if (!hasTimeShift) { + return allTimeFields + .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter); + } + return [ + { + meta: { index: indexPattern?.id, params: {}, alias: '', disabled: false, negate: false }, + query: { + bool: { + should: [ + ...Object.entries(timeShifts).map(([, shift]) => { + return { + bool: { + filter: allTimeFields + .map((fieldName) => + getTime( + indexPattern, + { + from: moment(timeRange.from).subtract(shift).toISOString(), + to: moment(timeRange.to).subtract(shift).toISOString(), + }, + { fieldName, forceNow } + ) + ) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }; + }), + { + bool: { + filter: allTimeFields + .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + ]; }); } diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 585126e1184d2..2a7f4fad83bb5 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -75,7 +75,13 @@ import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; -import { AggConfigs, ES_SEARCH_STRATEGY, ISearchGeneric, ISearchOptions } from '../..'; +import { + AggConfigs, + ES_SEARCH_STRATEGY, + IEsSearchResponse, + ISearchGeneric, + ISearchOptions, +} from '../..'; import type { ISearchSource, SearchFieldValue, @@ -408,6 +414,15 @@ export class SearchSource { } } + private postFlightTransform(response: IEsSearchResponse) { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + return aggs.postFlightTransform(response); + } else { + return response; + } + } + private async fetchOthers(response: estypes.SearchResponse, options: ISearchOptions) { const aggs = this.getField('aggs'); if (aggs instanceof AggConfigs) { @@ -445,24 +460,26 @@ export class SearchSource { if (isErrorResponse(response)) { obs.error(response); } else if (isPartialResponse(response)) { - obs.next(response); + obs.next(this.postFlightTransform(response)); } else { if (!this.hasPostFlightRequests()) { - obs.next(response); + obs.next(this.postFlightTransform(response)); obs.complete(); } else { // Treat the complete response as partial, then run the postFlightRequests. obs.next({ - ...response, + ...this.postFlightTransform(response), isPartial: true, isRunning: true, }); const sub = from(this.fetchOthers(response.rawResponse, options)).subscribe({ next: (responseWithOther) => { - obs.next({ - ...response, - rawResponse: responseWithOther, - }); + obs.next( + this.postFlightTransform({ + ...response, + rawResponse: responseWithOther!, + }) + ); }, error: (e) => { obs.error(e); diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index 4a8972d4384c2..4ae8a192bc574 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -139,7 +139,7 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { ...esResponse.aggregations, - doc_count: esResponse.hits?.total, + doc_count: esResponse.aggregations.doc_count || esResponse.hits?.total, }; collectBucket(aggConfigs, write, topLevelBucket, '', 1); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index a54901a2a2fe1..3c959693d45ae 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -45,7 +45,7 @@ export function FrameLayout(props: FrameLayoutProps) { {props.workspacePanel} - {props.suggestionsPanel} + {/* {props.suggestionsPanel} */}
( 'aggFilteredMetric', From 4c4ad720fa2ca442e56b0b342138bb5d31bc52aa Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 29 Apr 2021 17:59:07 +0200 Subject: [PATCH 02/31] stabilize response merging and start building UI --- .../data/common/search/aggs/agg_configs.ts | 97 ++++++++++--- .../data/common/search/aggs/buckets/terms.ts | 45 ++++++ .../data/common/search/tabify/tabify.ts | 2 +- .../dimension_panel/time_shift.tsx | 131 ++++++++++++++++++ .../definitions/calculations/counter_rate.tsx | 2 + .../calculations/cumulative_sum.tsx | 2 + .../definitions/calculations/differences.tsx | 2 + .../calculations/moving_average.tsx | 2 + .../operations/definitions/cardinality.tsx | 2 + .../operations/definitions/column_types.ts | 1 + .../operations/definitions/count.tsx | 2 + .../operations/definitions/index.ts | 1 + .../operations/definitions/last_value.tsx | 2 + .../operations/definitions/metrics.tsx | 2 + .../operations/definitions/percentile.tsx | 2 + 15 files changed, 271 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index eeb370765a77b..b5ab9ab9b891e 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -10,7 +10,12 @@ import moment from 'moment'; import _, { cloneDeep, isArray } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; -import { Aggregate, FiltersAggregate, FiltersBucketItem } from '@elastic/elasticsearch/api/types'; +import { + Aggregate, + Bucket, + FiltersAggregate, + FiltersBucketItem, +} from '@elastic/elasticsearch/api/types'; import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; @@ -50,6 +55,8 @@ export interface AggConfigsOptions { export type CreateAggConfigParams = Assign; +type GenericBucket = Bucket & { [property: string]: Aggregate }; + /** * @name AggConfigs * @@ -354,6 +361,19 @@ export class AggConfigs { return timeShifts; } + hasTimeShifts(): boolean { + const timeShifts: Record = {}; + this.getAll() + .filter((agg) => agg.schema === 'metric') + .map((agg) => agg.getTimeShift()) + .forEach((timeShift) => { + if (timeShift) { + timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } + }); + return Object.keys(timeShifts).length > 0; + } + isTimeShiftSplitAgg(aggConfig: AggConfig) { // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); @@ -372,41 +392,70 @@ export class AggConfigs { const transformedRawResponse = cloneDeep(response.rawResponse); const aggCursor = transformedRawResponse.aggregations!; - // TODO: Add a "context" param here which keeps track of at which level of the response we are - const mergeObject = (target: any, source: any, shiftKey: string) => { + const bucketAggs = this.aggs.filter((agg) => agg.type.type === AggGroupNames.Buckets); + + const mergeAggLevel = ( + target: GenericBucket, + source: GenericBucket, + shiftKey: string, + aggIndex: number + ) => { Object.entries(source).forEach(([key, val]) => { - if (typeof val !== 'object') { - if (key === 'doc_count') { - target[`doc_count_${shiftKey}`] = val; - } else { - target[key] = val; - } + // copy over doc count into special key + if (typeof val === 'number' && key === 'doc_count') { + target[`doc_count_${shiftKey}`] = val; + } else if (typeof val !== 'object') { + // other meta keys not of interest + return; } else { + // a sub-agg const agg = this.byId(key); if (agg && agg.type.type === 'metrics') { const timeShift = agg.getTimeShift(); - if (!timeShift || String(timeShift.asMilliseconds()) !== shiftKey) { - // this is a metric from another time shift, do not copy over - return; + if (timeShift && String(timeShift.asMilliseconds()) === shiftKey) { + // this is a metric from the current time shift, copy it over + target[key] = source[key]; + } + } else if (agg === bucketAggs[aggIndex]) { + // expected next bucket sub agg + const subAggregate = val as Aggregate; + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters + const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as + | GenericBucket[] + | Record + | undefined; + const baseBuckets = ('buckets' in baseSubAggregate + ? baseSubAggregate.buckets + : undefined) as GenericBucket[] | Record | undefined; + // merge + if (isArray(buckets) && isArray(baseBuckets)) { + buckets.forEach((bucket, index) => + mergeAggLevel(baseBuckets[index], bucket, shiftKey, aggIndex + 1) + ); + } else if (baseBuckets && buckets && !isArray(baseBuckets)) { + Object.entries(buckets).forEach(([bucketKey, bucket]) => + mergeAggLevel(baseBuckets[bucketKey], bucket, shiftKey, aggIndex + 1) + ); } } - if (!target[key]) { - target[key] = {}; - } - mergeObject(target[key], source[key], shiftKey); } }); }; - const transformTimeShift = (cursor: Record): undefined => { + const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { if (cursor.time_offset_split) { const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< string, FiltersBucketItem >; const subTree = timeShiftedBuckets['0']; - // TODO pull in metrics from other buckets - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeObject(subTree, timeShiftedBuckets[key], key); + Object.keys(timeShifts).forEach((key) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + key, + aggIndex + ); }); delete cursor.time_offset_split; @@ -420,13 +469,15 @@ export class AggConfigs { return; } if (isArray(subAgg.buckets)) { - subAgg.buckets.forEach(transformTimeShift); + subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); } else { - Object.values(subAgg.buckets).forEach(transformTimeShift); + Object.values(subAgg.buckets).forEach((bucket) => + transformTimeShift(bucket, aggIndex + 1) + ); } }); }; - transformTimeShift(aggCursor); + transformTimeShift(aggCursor, 0); return { ...response, rawResponse: transformedRawResponse, diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 1b876051d009b..44088ecb49db6 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -9,6 +9,7 @@ import { noop } from 'lodash'; import { i18n } from '@kbn/i18n'; +import moment from 'moment'; import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { createFilterTerms } from './create_filter/terms'; @@ -179,6 +180,50 @@ export const getTermsBucketAgg = () => return; } + if (aggs?.hasTimeShifts()) { + const shift = orderAgg.getTimeShift(); + orderAgg = aggs.createAggConfig( + { + type: 'filtered_metric', + id: 'shifted-order', + params: { + customBucket: aggs + .createAggConfig( + { + type: 'filter', + id: 'shift', + params: { + filter: { + language: 'lucene', + query: { + range: { + [aggs.timeFields![0]]: { + from: moment(aggs.timeRange!.from) + .subtract(shift || 0) + .toISOString(), + to: moment(aggs.timeRange!.to) + .subtract(shift || 0) + .toISOString(), + }, + }, + }, + }, + }, + }, + { + addToAggConfigs: false, + } + ) + .serialize(), + customMetric: orderAgg.serialize(), + }, + enabled: false, + }, + { + addToAggConfigs: false, + } + ); + } if (orderAgg.type.name === 'count') { order._count = dir; return; diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index 4ae8a192bc574..a4d9551da75d5 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -139,7 +139,7 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { ...esResponse.aggregations, - doc_count: esResponse.aggregations.doc_count || esResponse.hits?.total, + doc_count: esResponse.aggregations?.doc_count || esResponse.hits?.total, }; collectBucket(aggConfigs, write, topLevelBucket, '', 1); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx new file mode 100644 index 0000000000000..65bc23b4eb1ca --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButtonIcon, EuiLink, EuiPanel, EuiPopover } from '@elastic/eui'; +import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useState } from 'react'; +import { Query } from 'src/plugins/data/public'; +import { IndexPatternColumn, operationDefinitionMap } from '../operations'; +import { isQueryValid } from '../operations/definitions/filters'; +import { QueryInput } from '../query_input'; +import { IndexPattern, IndexPatternLayer } from '../types'; + +// to do: get the language from uiSettings +export const defaultFilter: Query = { + query: '', + language: 'kuery', +}; + +export function setFilter(columnId: string, layer: IndexPatternLayer, query: Query | undefined) { + return { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + ...layer.columns[columnId], + filter: query, + }, + }, + }; +} + +export function Filtering({ + selectedColumn, + columnId, + layer, + updateLayer, + indexPattern, + isInitiallyOpen, +}: { + selectedColumn: IndexPatternColumn; + indexPattern: IndexPattern; + columnId: string; + layer: IndexPatternLayer; + updateLayer: (newLayer: IndexPatternLayer) => void; + isInitiallyOpen: boolean; +}) { + const [filterPopoverOpen, setFilterPopoverOpen] = useState(isInitiallyOpen); + const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; + if (!selectedOperation.filterable || !selectedColumn.filter) { + return null; + } + + const isInvalid = !isQueryValid(selectedColumn.filter, indexPattern); + + return ( + + + + { + setFilterPopoverOpen(false); + }} + anchorClassName="eui-fullWidth" + panelClassName="lnsIndexPatternDimensionEditor__filtersEditor" + button={ + + + {/* Empty for spacing */} + + { + setFilterPopoverOpen(!filterPopoverOpen); + }} + color={isInvalid ? 'danger' : 'text'} + title={i18n.translate('xpack.lens.indexPattern.filterBy.clickToEdit', { + defaultMessage: 'Click to edit', + })} + > + {selectedColumn.filter.query || + i18n.translate('xpack.lens.indexPattern.filterBy.emptyFilterQuery', { + defaultMessage: '(empty)', + })} + + + + { + updateLayer(setFilter(columnId, layer, undefined)); + }} + iconType="cross" + /> + + + + } + > + { + updateLayer(setFilter(columnId, layer, newQuery)); + }} + isInvalid={false} + onSubmit={() => {}} + /> + + + + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index c57f70ba1b58b..c3c6dd92f1781 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -92,6 +92,7 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale, + timeShift: previousColumn?.timeShift, filter: previousColumn?.filter, params: getFormatFromPreviousColumn(previousColumn), }; @@ -118,4 +119,5 @@ export const counterRateOperation: OperationDefinition< }, timeScalingMode: 'mandatory', filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 7cec1fa0d4bbc..aefd6737f1bea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -86,6 +86,7 @@ export const cumulativeSumOperation: OperationDefinition< isBucketed: false, scale: 'ratio', filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, references: referenceIds, params: getFormatFromPreviousColumn(previousColumn), }; @@ -111,4 +112,5 @@ export const cumulativeSumOperation: OperationDefinition< )?.join(', '); }, filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index bef3fbc2e48ae..2143dde0cb44a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -82,6 +82,7 @@ export const derivativeOperation: OperationDefinition< references: referenceIds, timeScale: previousColumn?.timeScale, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: getFormatFromPreviousColumn(previousColumn), }; }, @@ -108,4 +109,5 @@ export const derivativeOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 46cc64c2bc518..72dbcc983f539 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -90,6 +90,7 @@ export const movingAverageOperation: OperationDefinition< references: referenceIds, timeScale: previousColumn?.timeScale, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: { window: 5, ...getFormatFromPreviousColumn(previousColumn), @@ -121,6 +122,7 @@ export const movingAverageOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + shiftable: true, }; function MovingAverageParamEditor({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index fa1691ba9a78e..0195304542385 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -71,6 +71,7 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), buildColumn({ field, previousColumn }) { return { @@ -81,6 +82,7 @@ export const cardinalityOperation: OperationDefinition { */ timeScalingMode?: TimeScalingMode; filterable?: boolean; + shiftable?: boolean; getHelpMessage?: (props: HelpProps) => React.ReactNode; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 4f5c897fb5378..d10c2fdbd73f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -162,6 +162,7 @@ export const lastValueOperation: OperationDefinition { return buildExpressionFunction('aggTopHit', { id: columnId, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 20580634d12e6..fa9b5876c566f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -99,6 +99,7 @@ function buildMetricOperation>({ scale: 'ratio', timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: getFormatFromPreviousColumn(previousColumn), } as T), onFieldChange: (oldColumn, field) => { @@ -119,6 +120,7 @@ function buildMetricOperation>({ getErrorMessage: (layer, columnId, indexPattern) => getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), filterable: true, + shiftable: true, } as OperationDefinition; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index dd0f3b978da5f..12172c407dda9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -52,6 +52,7 @@ export const percentileOperation: OperationDefinition { if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { return { @@ -88,6 +89,7 @@ export const percentileOperation: OperationDefinition Date: Fri, 30 Apr 2021 14:29:55 +0200 Subject: [PATCH 03/31] continue building time shift --- .../data/common/search/aggs/agg_config.ts | 37 ++- .../data/common/search/aggs/agg_configs.ts | 219 ++++++++++++----- .../data/common/search/aggs/agg_type.ts | 23 +- .../search/aggs/buckets/bucket_agg_type.ts | 27 ++- .../search/aggs/buckets/date_histogram.ts | 11 + .../search/aggs/metrics/cardinality_fn.ts | 7 + .../search/aggs/metrics/metric_agg_type.ts | 5 + .../expressions/esaggs/request_handler.ts | 62 +---- .../public/components/agg_params_helper.ts | 3 + .../dimension_panel/dimension_editor.tsx | 28 +++ .../dimension_panel/time_shift.tsx | 222 ++++++++++++------ .../operations/definitions/cardinality.tsx | 1 + .../operations/definitions/count.tsx | 1 + .../operations/definitions/last_value.tsx | 1 + .../operations/definitions/metrics.tsx | 1 + .../operations/definitions/percentile.tsx | 1 + .../operations/layer_helpers.ts | 1 + .../indexpattern_datasource/to_expression.ts | 19 +- 18 files changed, 463 insertions(+), 206 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 42dc52c59d822..38bc552bddbe0 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -20,7 +20,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; -import { IAggConfigs } from './agg_configs'; +import { GenericBucket, IAggConfigs } from './agg_configs'; type State = string | number | boolean | null | undefined | SerializableState; @@ -173,13 +173,48 @@ export class AggConfig { return _.get(this.params, key); } + hasTimeShift(): boolean { + return Boolean(this.getParam('timeShift')); + } + getTimeShift(): undefined | moment.Duration { const rawTimeShift = this.getParam('timeShift'); if (!rawTimeShift) return undefined; + if (rawTimeShift === 'previous') { + const timeShiftInterval = this.aggConfigs.getTimeShiftInterval(); + if (timeShiftInterval) { + return timeShiftInterval; + } else if (!this.aggConfigs.timeRange) { + return; + } + return moment.duration( + moment(this.aggConfigs.timeRange.to).diff(this.aggConfigs.timeRange.from) + ); + } const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); return moment.duration(Number(amount), unit); } + getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { + return this.type.getShiftedKey(this, key, timeShift); + } + + getTimeShiftInterval(): undefined | moment.Duration { + return this.type.getTimeShiftInterval(this); + } + + splitForTimeShift(aggs: IAggConfigs) { + return this.type.splitForTimeShift(this, aggs); + } + + orderBuckets(a: GenericBucket, b: GenericBucket): number { + if (this.type.orderBuckets) { + return this.type.orderBuckets(this, a, b); + } else { + return 0; + } + } + write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index b5ab9ab9b891e..afd537321e903 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -15,15 +15,21 @@ import { Bucket, FiltersAggregate, FiltersBucketItem, + MultiBucketAggregate, } from '@elastic/elasticsearch/api/types'; -import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; +import { + IEsSearchResponse, + ISearchOptions, + ISearchSource, + RangeFilter, +} from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; import { IAggType } from './agg_type'; import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; -import { TimeRange } from '../../../common'; +import { TimeRange, getTime, isRangeFilter } from '../../../common'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -55,7 +61,7 @@ export interface AggConfigsOptions { export type CreateAggConfigParams = Assign; -type GenericBucket = Bucket & { [property: string]: Aggregate }; +export type GenericBucket = Bucket & { [property: string]: Aggregate }; /** * @name AggConfigs @@ -270,40 +276,50 @@ export class AggConfigs { timeShifts: Record, dslLvlCursor: Record ) { - if (this.isTimeShiftSplitAgg(config)) { - const filters: Record = {}; - // TODO validate this - const timeField = this.timeFields![0]; - filters['0'] = { + if (!config.splitForTimeShift(this)) { + return dslLvlCursor; + } + if (!this.timeFields || this.timeFields.length < 1) { + throw new Error('Time shift can only be used with configured time field'); + } + if (!this.timeRange) { + throw new Error('Time shift can only be used with configured time range'); + } + const timeRange = this.timeRange; + const filters: Record = {}; + const timeField = this.timeFields[0]; + filters['0'] = { + range: { + [timeField]: { + // only works if there is a time range + gte: timeRange.from, + lte: timeRange.to, + }, + }, + }; + const timeShiftInterval = config.getTimeShiftInterval(); + Object.entries(timeShifts).forEach(([key, shift]) => { + if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { + throw new Error('All time shifts need to be larger than underlying date interval'); + } + filters[key] = { range: { [timeField]: { // only works if there is a time range - gte: this.timeRange!.from, - lte: this.timeRange!.to, + gte: moment(timeRange.from).subtract(shift).toISOString(), + lte: moment(timeRange.to).subtract(shift).toISOString(), }, }, }; - Object.entries(timeShifts).forEach(([key, shift]) => { - filters[key] = { - range: { - [timeField]: { - // only works if there is a time range - gte: moment(this.timeRange!.from).subtract(shift).toISOString(), - lte: moment(this.timeRange!.to).subtract(shift).toISOString(), - }, - }, - }; - }); - dslLvlCursor.time_offset_split = { - filters: { - filters, - }, - aggs: {}, - }; + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; - dslLvlCursor = dslLvlCursor.time_offset_split.aggs; - } - return dslLvlCursor; + return dslLvlCursor.time_offset_split.aggs; } getAll() { @@ -361,27 +377,81 @@ export class AggConfigs { return timeShifts; } + getTimeShiftInterval(): moment.Duration | undefined { + const splitAgg = this.getAll().find((agg) => agg.splitForTimeShift(this)); + return splitAgg?.getTimeShiftInterval(); + } + hasTimeShifts(): boolean { - const timeShifts: Record = {}; - this.getAll() - .filter((agg) => agg.schema === 'metric') - .map((agg) => agg.getTimeShift()) - .forEach((timeShift) => { - if (timeShift) { - timeShifts[String(timeShift.asMilliseconds())] = timeShift; - } - }); - return Object.keys(timeShifts).length > 0; + return this.getAll().some((agg) => agg.hasTimeShift()); } - isTimeShiftSplitAgg(aggConfig: AggConfig) { - // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here - const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); - // the first date histogram or the first metric (if there are no date histograms) is the level to add the time shift filter split - return ( - (hasDateHistograms && aggConfig.type.name === 'date_histogram') || - this.byType('metrics')[0] === aggConfig - ); + getSearchSourceTimeFilter(forceNow?: Date) { + if (!this.timeFields || !this.timeRange) { + return []; + } + const timeRange = this.timeRange; + const timeFields = this.timeFields; + const timeShifts = this.getTimeShifts(); + const hasTimeShift = Object.values(this.getTimeShifts()).length > 0; + if (!hasTimeShift) { + return this.timeFields + .map((fieldName) => getTime(this.indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter); + } + return [ + { + meta: { + index: this.indexPattern?.id, + params: {}, + alias: '', + disabled: false, + negate: false, + }, + query: { + bool: { + should: [ + ...Object.entries(timeShifts).map(([, shift]) => { + return { + bool: { + filter: timeFields + .map( + (fieldName) => + [ + getTime(this.indexPattern, timeRange, { fieldName, forceNow }), + fieldName, + ] as [RangeFilter | undefined, string] + ) + .filter(([filter]) => isRangeFilter(filter)) + .map(([filter, field]) => ({ + range: { + [field]: { + gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), + lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), + }, + }, + })), + }, + }; + }), + { + bool: { + filter: timeFields + .map((fieldName) => + getTime(this.indexPattern, timeRange, { fieldName, forceNow }) + ) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + ]; } postFlightTransform(response: IEsSearchResponse) { @@ -397,46 +467,71 @@ export class AggConfigs { const mergeAggLevel = ( target: GenericBucket, source: GenericBucket, - shiftKey: string, + shift: moment.Duration, aggIndex: number ) => { Object.entries(source).forEach(([key, val]) => { // copy over doc count into special key if (typeof val === 'number' && key === 'doc_count') { - target[`doc_count_${shiftKey}`] = val; + target[`doc_count_${shift.asMilliseconds()}`] = val; } else if (typeof val !== 'object') { // other meta keys not of interest return; } else { // a sub-agg const agg = this.byId(key); - if (agg && agg.type.type === 'metrics') { + if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); - if (timeShift && String(timeShift.asMilliseconds()) === shiftKey) { + if (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) { // this is a metric from the current time shift, copy it over target[key] = source[key]; } } else if (agg === bucketAggs[aggIndex]) { // expected next bucket sub agg const subAggregate = val as Aggregate; - const baseSubAggregate = target[key] as Aggregate; - // only supported bucket formats in agg configs are array of buckets and record of buckets for filters const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as | GenericBucket[] | Record | undefined; + if (!target[key]) { + // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate + // which will be filled with shifted data + target[key] = { + buckets: isArray(buckets) ? [] : {}, + }; + } + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters const baseBuckets = ('buckets' in baseSubAggregate ? baseSubAggregate.buckets : undefined) as GenericBucket[] | Record | undefined; // merge if (isArray(buckets) && isArray(baseBuckets)) { - buckets.forEach((bucket, index) => - mergeAggLevel(baseBuckets[index], bucket, shiftKey, aggIndex + 1) - ); + const baseBucketMap: Record = {}; + baseBuckets.forEach((bucket) => { + baseBucketMap[String(bucket.key)] = bucket; + }); + buckets.forEach((bucket) => { + const bucketKey = agg.getShiftedKey(bucket.key, shift); + // if a bucket is missing in the map, create an empty one + if (!baseBucketMap[bucketKey]) { + baseBucketMap[String(bucketKey)] = { + key: bucketKey, + } as GenericBucket; + } + mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); + }); + (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( + baseBucketMap + ).sort(agg.orderBuckets.bind(agg)); } else if (baseBuckets && buckets && !isArray(baseBuckets)) { - Object.entries(buckets).forEach(([bucketKey, bucket]) => - mergeAggLevel(baseBuckets[bucketKey], bucket, shiftKey, aggIndex + 1) - ); + Object.entries(buckets).forEach(([bucketKey, bucket]) => { + // if a bucket is missing in the base response, create an empty one + if (!baseBuckets[bucketKey]) { + baseBuckets[bucketKey] = {} as GenericBucket; + } + mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); + }); } } } @@ -449,11 +544,11 @@ export class AggConfigs { FiltersBucketItem >; const subTree = timeShiftedBuckets['0']; - Object.keys(timeShifts).forEach((key) => { + Object.entries(timeShifts).forEach(([key, shift]) => { mergeAggLevel( subTree as GenericBucket, timeShiftedBuckets[key] as GenericBucket, - key, + shift, aggIndex ); }); diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index f0f3912bf64fe..4d30d6a211650 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -8,6 +8,7 @@ import { constant, noop, identity } from 'lodash'; import { i18n } from '@kbn/i18n'; +import moment from 'moment'; import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; @@ -16,7 +17,7 @@ import type { RequestAdapter } from 'src/plugins/inspector/common'; import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; -import { IAggConfigs } from './agg_configs'; +import { GenericBucket, IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; @@ -215,6 +216,26 @@ export class AggType< return agg.id; }; + getShiftedKey( + agg: TAggConfig, + key: string | number, + timeShift: moment.Duration + ): string | number { + return key; + } + + splitForTimeShift(agg: TAggConfig, aggs: IAggConfigs) { + return false; + } + + getTimeShiftInterval(agg: TAggConfig): undefined | moment.Duration { + return undefined; + } + + orderBuckets(agg: TAggConfig, a: GenericBucket, b: GenericBucket): number { + return Number(a.key) - Number(b.key); + } + /** * Generic AggType Constructor * diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts index e9ed3799b90cf..1626a89e9a331 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts @@ -6,8 +6,9 @@ * Side Public License, v 1. */ +import moment from 'moment'; import { IAggConfig } from '../agg_config'; -import { KBN_FIELD_TYPES } from '../../../../common'; +import { GenericBucket, IAggConfigs, KBN_FIELD_TYPES } from '../../../../common'; import { AggType, AggTypeConfig } from '../agg_type'; import { AggParamType } from '../param_types/agg'; @@ -26,6 +27,14 @@ const bucketType = 'buckets'; interface BucketAggTypeConfig extends AggTypeConfig> { getKey?: (bucket: any, key: any, agg: IAggConfig) => any; + getShiftedKey?: ( + agg: TBucketAggConfig, + key: string | number, + timeShift: moment.Duration + ) => string | number; + orderBuckets?(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number; + splitForTimeShift?(agg: TBucketAggConfig, aggs: IAggConfigs): boolean; + getTimeShiftInterval?(agg: TBucketAggConfig): undefined | moment.Duration; } export class BucketAggType extends AggType< @@ -43,6 +52,22 @@ export class BucketAggType { return key || bucket.key; }); + + if (config.getShiftedKey) { + this.getShiftedKey = config.getShiftedKey; + } + + if (config.orderBuckets) { + this.orderBuckets = config.orderBuckets; + } + + if (config.getTimeShiftInterval) { + this.getTimeShiftInterval = config.getTimeShiftInterval; + } + + if (config.splitForTimeShift) { + this.splitForTimeShift = config.splitForTimeShift; + } } } diff --git a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts index 4a83ae38d34db..4cbf6562487b2 100644 --- a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts +++ b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts @@ -135,6 +135,17 @@ export const getDateHistogramBucketAgg = ({ }, }; }, + getShiftedKey(agg, key, timeShift) { + return moment(key).add(timeShift).valueOf(); + }, + splitForTimeShift(agg, aggs) { + return aggs.hasTimeShifts() && Boolean(aggs.timeFields?.includes(agg.fieldName())); + }, + getTimeShiftInterval(agg) { + const { useNormalizedEsInterval } = agg.params; + const interval = agg.buckets.getInterval(useNormalizedEsInterval); + return interval; + }, params: [ { name: 'field', diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts index ee0f72e01e1de..fe559b71094ec 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts @@ -67,6 +67,13 @@ export const aggCardinality = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 45bc571c37515..da71383dd564c 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -12,6 +12,7 @@ import { AggParamType } from '../param_types/agg'; import { AggConfig } from '../agg_config'; import { METRIC_TYPES } from './metric_agg_types'; import { BaseParamType, FieldTypes } from '../param_types'; +import { AggGroupNames } from '../agg_groups'; export interface IMetricAggConfig extends AggConfig { type: InstanceType; @@ -77,6 +78,10 @@ export class MetricAggType false); + + // split at this point if there are time shifts and this is the first metric + this.splitForTimeShift = (agg, aggs) => + aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg; } } diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index cf20e75f12a53..1eefd71d9e212 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -7,18 +7,9 @@ */ import { i18n } from '@kbn/i18n'; -import moment from 'moment'; import { Adapters } from 'src/plugins/inspector/common'; -import { - calculateBounds, - Filter, - getTime, - IndexPattern, - isRangeFilter, - Query, - TimeRange, -} from '../../../../common'; +import { calculateBounds, Filter, IndexPattern, Query, TimeRange } from '../../../../common'; import { IAggConfigs } from '../../aggs'; import { ISearchStartSearchSource } from '../../search_source'; @@ -101,56 +92,7 @@ export const handleRequest = async ({ // filters for that those time fields if (timeRange && allTimeFields.length > 0) { timeFilterSearchSource.setField('filter', () => { - const timeShifts = aggs.getTimeShifts(); - const hasTimeShift = Object.values(aggs.getTimeShifts()).length > 0; - if (!hasTimeShift) { - return allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter); - } - return [ - { - meta: { index: indexPattern?.id, params: {}, alias: '', disabled: false, negate: false }, - query: { - bool: { - should: [ - ...Object.entries(timeShifts).map(([, shift]) => { - return { - bool: { - filter: allTimeFields - .map((fieldName) => - getTime( - indexPattern, - { - from: moment(timeRange.from).subtract(shift).toISOString(), - to: moment(timeRange.to).subtract(shift).toISOString(), - }, - { fieldName, forceNow } - ) - ) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, - })), - }, - }; - }), - { - bool: { - filter: allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, - })), - }, - }, - ], - minimum_should_match: 1, - }, - }, - }, - ]; + return aggs.getSearchSourceTimeFilter(forceNow); }); } diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index fcd97de56fc65..55db880a83932 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -72,6 +72,9 @@ function getAggParamsToRender({ if (hideCustomLabel && param.name === 'customLabel') { return; } + if (param.name === 'timeShift') { + return; + } // if field param exists, compute allowed fields if (param.type === 'field') { let availableFields: IndexPatternField[] = (param as IFieldParamType).getAvailableFields(agg); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 2c503a7bd6967..ae565720e098e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -45,6 +45,7 @@ import { ReferenceEditor } from './reference_editor'; import { setTimeScaling, TimeScaling } from './time_scaling'; import { defaultFilter, Filtering, setFilter } from './filtering'; import { AdvancedOptions } from './advanced_options'; +import { setTimeShift, TimeShift } from './time_shift'; const operationPanels = getOperationDisplay(); @@ -160,6 +161,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }, [fieldByOperation, operationWithoutField]); const [filterByOpenInitially, setFilterByOpenInitally] = useState(false); + const [timeShiftedFocused, setTimeShiftFocused] = useState(false); // Operations are compatible if they match inputs. They are always compatible in // the empty state. Field-based operations are not compatible with field-less operations. @@ -519,6 +521,32 @@ export function DimensionEditor(props: DimensionEditorProps) { /> ) : null, }, + { + title: i18n.translate('xpack.lens.indexPattern.timeShift.label', { + defaultMessage: 'Shift in time', + }), + dataTestSubj: 'indexPattern-time-shift-enable', + onClick: () => { + setTimeShiftFocused(true); + setStateWrapper(setTimeShift(columnId, state.layers[layerId], '')); + }, + showInPopover: Boolean( + operationDefinitionMap[selectedColumn.operationType].shiftable && + selectedColumn.timeShift === undefined + ), + inlineElement: + operationDefinitionMap[selectedColumn.operationType].shiftable && + selectedColumn.timeShift !== undefined ? ( + + ) : null, + }, ]} /> )} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 65bc23b4eb1ca..c81d9e97f2dfd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -5,14 +5,13 @@ * 2.0. */ -import { EuiButtonIcon, EuiLink, EuiPanel, EuiPopover } from '@elastic/eui'; +import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; +import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React, { useState } from 'react'; +import React from 'react'; import { Query } from 'src/plugins/data/public'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; -import { isQueryValid } from '../operations/definitions/filters'; -import { QueryInput } from '../query_input'; import { IndexPattern, IndexPatternLayer } from '../types'; // to do: get the language from uiSettings @@ -21,111 +20,178 @@ export const defaultFilter: Query = { language: 'kuery', }; -export function setFilter(columnId: string, layer: IndexPatternLayer, query: Query | undefined) { +export function setTimeShift( + columnId: string, + layer: IndexPatternLayer, + timeShift: string | undefined +) { return { ...layer, columns: { ...layer.columns, [columnId]: { ...layer.columns[columnId], - filter: query, + timeShift, }, }, }; } -export function Filtering({ +const timeShiftOptions = [ + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { defaultMessage: '1 hour' }), + value: '1h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.3hours', { + defaultMessage: '3 hours', + }), + value: '3h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.6hours', { + defaultMessage: '6 hours', + }), + value: '6h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.12hours', { + defaultMessage: '12 hours', + }), + value: '12h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { defaultMessage: '1 day' }), + value: '1d', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { defaultMessage: '1 week' }), + value: '1w', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { defaultMessage: '1 month' }), + value: '1M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.3months', { + defaultMessage: '3 months', + }), + value: '3M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.6months', { + defaultMessage: '6 months', + }), + value: '6M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { defaultMessage: '1 year' }), + value: '1y', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.previous', { + defaultMessage: 'Previous', + }), + value: 'previous', + }, +]; + +export function TimeShift({ selectedColumn, columnId, layer, updateLayer, indexPattern, - isInitiallyOpen, + isFocused, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; columnId: string; layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; - isInitiallyOpen: boolean; + isFocused: boolean; }) { - const [filterPopoverOpen, setFilterPopoverOpen] = useState(isInitiallyOpen); const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; - if (!selectedOperation.filterable || !selectedColumn.filter) { + if (!selectedOperation.shiftable || selectedColumn.timeShift === undefined) { return null; } - const isInvalid = !isQueryValid(selectedColumn.filter, indexPattern); + const timeShift = selectedColumn.timeShift; + + function getSelectedOption() { + if (timeShift === '') return []; + const goodPick = timeShiftOptions.filter(({ value }) => value === timeShift); + if (goodPick.length > 0) return goodPick; + return [ + { + value: timeShift, + label: timeShift, + }, + ]; + } return ( - { + if (r && isFocused) { + const timeShiftInput = r.querySelector('[data-test-subj="comboBoxSearchInput"]'); + if (timeShiftInput instanceof HTMLInputElement) { + timeShiftInput.focus(); + } + } + }} > - - - { - setFilterPopoverOpen(false); - }} - anchorClassName="eui-fullWidth" - panelClassName="lnsIndexPatternDimensionEditor__filtersEditor" - button={ - - - {/* Empty for spacing */} - - { - setFilterPopoverOpen(!filterPopoverOpen); - }} - color={isInvalid ? 'danger' : 'text'} - title={i18n.translate('xpack.lens.indexPattern.filterBy.clickToEdit', { - defaultMessage: 'Click to edit', - })} - > - {selectedColumn.filter.query || - i18n.translate('xpack.lens.indexPattern.filterBy.emptyFilterQuery', { - defaultMessage: '(empty)', - })} - - - - { - updateLayer(setFilter(columnId, layer, undefined)); - }} - iconType="cross" - /> - - - - } - > - { - updateLayer(setFilter(columnId, layer, newQuery)); + + + + { + updateLayer(setTimeShift(columnId, layer, val)); + }} + onChange={(choices) => { + if (choices.length === 0) { + updateLayer(setTimeShift(columnId, layer, '')); + return; + } + + const choice = choices[0].value as string; + updateLayer(setTimeShift(columnId, layer, choice)); }} - isInvalid={false} - onSubmit={() => {}} /> - - - - + + {selectedOperation.timeScalingMode === 'optional' && ( + + { + updateLayer(setTimeShift(columnId, layer, undefined)); + }} + iconType="cross" + /> + + )} + + + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 0195304542385..26318b643369e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -92,6 +92,7 @@ export const cardinalityOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index a3272064bd602..6790019803343 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -83,6 +83,7 @@ export const countOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index d10c2fdbd73f6..5f4f07e97afdd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -181,6 +181,7 @@ export const lastValueOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, + timeShift: column.timeShift, }).toAst(); }, getErrorMessage: (layer, columnId, indexPattern) => diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 12172c407dda9..8bc99f078fe37 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -112,6 +112,7 @@ export const percentileOperation: OperationDefinition { + const referencedColumn = columns[referenceColumnId]; + const referenceDef = operationDefinitionMap[column.operationType]; + if (referenceDef.shiftable) { + columns[referenceColumnId] = { ...referencedColumn, timeShift: column.timeShift }; + } + }); + } }); const columnEntries = columnOrder.map((colId) => [colId, columns[colId]] as const); @@ -84,9 +100,6 @@ function getExpressionForLayer( layer, uiSettings ); - if (col.label === 'shifted') { - aggAst.arguments.timeShift = ['1h']; - } if (wrapInFilter) { aggAst = buildExpressionFunction( 'aggFilteredMetric', From fa12d2e0889fdd938596d97e4a648af3dadd0365 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 30 Apr 2021 18:33:12 +0200 Subject: [PATCH 04/31] add better error handling --- .../data/common/search/aggs/agg_config.ts | 10 +- .../data/common/search/aggs/utils/index.ts | 1 + .../search/aggs/utils/parse_time_shift.ts | 28 +++++ .../dimension_panel/dimension_editor.tsx | 2 + .../dimension_panel/time_shift.tsx | 103 ++++++++++++++--- .../operations/definitions/date_histogram.tsx | 107 ++++++++++++++++-- .../operations/definitions/index.ts | 2 + .../operations/definitions/terms/index.tsx | 32 +++++- .../operations/layer_helpers.ts | 2 + .../public/indexpattern_datasource/utils.ts | 8 +- 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/parse_time_shift.ts diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 38bc552bddbe0..5c34e17fb5c0b 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -21,6 +21,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; import { GenericBucket, IAggConfigs } from './agg_configs'; +import { parseTimeShift } from './utils'; type State = string | number | boolean | null | undefined | SerializableState; @@ -180,7 +181,11 @@ export class AggConfig { getTimeShift(): undefined | moment.Duration { const rawTimeShift = this.getParam('timeShift'); if (!rawTimeShift) return undefined; - if (rawTimeShift === 'previous') { + const parsedTimeShift = parseTimeShift(rawTimeShift); + if (parsedTimeShift === 'invalid') { + throw new Error(`could not parse time shift ${rawTimeShift}`); + } + if (parsedTimeShift === 'previous') { const timeShiftInterval = this.aggConfigs.getTimeShiftInterval(); if (timeShiftInterval) { return timeShiftInterval; @@ -191,8 +196,7 @@ export class AggConfig { moment(this.aggConfigs.timeRange.to).diff(this.aggConfigs.timeRange.from) ); } - const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); - return moment.duration(Number(amount), unit); + return parsedTimeShift; } getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index c92653e843233..1f2d0a308f0dd 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -15,3 +15,4 @@ export * from './ipv4_address'; export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; +export * from './parse_time_shift'; diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts new file mode 100644 index 0000000000000..e874f489b13f2 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; + +const allowedUnits = ['s', 'm', 'h', 'd', 'w', 'M', 'y'] as const; +type AllowedUnit = typeof allowedUnits[number]; + +/** + * This method parses a string into a time shift duration. + * If parsing fails, 'invalid' is returned. + * Allowed values are the string 'previous' and an integer followed by the units s,m,h,d,w,M,y + * */ +export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'invalid' => { + if (val === 'previous') { + return 'previous'; + } + const [, amount, unit] = val.match(/(\d+)(\w)/) || []; + const parsedAmount = Number(amount); + if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { + return 'invalid'; + } + return moment.duration(Number(amount), unit as AllowedUnit); +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ae565720e098e..73fdb6fe99ec8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -544,6 +544,8 @@ export function DimensionEditor(props: DimensionEditorProps) { layer={state.layers[layerId]} updateLayer={setStateWrapper} isFocused={timeShiftedFocused} + activeData={props.activeData} + layerId={layerId} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index c81d9e97f2dfd..8db0af5a71f0e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -9,10 +9,13 @@ import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; +import { search } from '../../../../../../src/plugins/data/public'; +import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; +import { IndexPatternDimensionEditorProps } from './dimension_panel'; // to do: get the language from uiSettings export const defaultFilter: Query = { @@ -39,53 +42,63 @@ export function setTimeShift( const timeShiftOptions = [ { - label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { defaultMessage: '1 hour' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { + defaultMessage: '1 hour (1h)', + }), value: '1h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.3hours', { - defaultMessage: '3 hours', + defaultMessage: '3 hours (3h)', }), value: '3h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.6hours', { - defaultMessage: '6 hours', + defaultMessage: '6 hours (6h)', }), value: '6h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.12hours', { - defaultMessage: '12 hours', + defaultMessage: '12 hours (12h)', }), value: '12h', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { defaultMessage: '1 day' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { + defaultMessage: '1 day (1d)', + }), value: '1d', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { defaultMessage: '1 week' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { + defaultMessage: '1 week (1w)', + }), value: '1w', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { defaultMessage: '1 month' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { + defaultMessage: '1 month (1M)', + }), value: '1M', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.3months', { - defaultMessage: '3 months', + defaultMessage: '3 months (3M)', }), value: '3M', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.6months', { - defaultMessage: '6 months', + defaultMessage: '6 months (6M)', }), value: '6M', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { defaultMessage: '1 year' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { + defaultMessage: '1 year (1y)', + }), value: '1y', }, { @@ -103,6 +116,8 @@ export function TimeShift({ updateLayer, indexPattern, isFocused, + activeData, + layerId, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -110,22 +125,52 @@ export function TimeShift({ layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; isFocused: boolean; + activeData: IndexPatternDimensionEditorProps['activeData']; + layerId: string; }) { + const [localValue, setLocalValue] = useState(selectedColumn.timeShift); + useEffect(() => { + setLocalValue(selectedColumn.timeShift); + }, [selectedColumn.timeShift]); const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; if (!selectedOperation.shiftable || selectedColumn.timeShift === undefined) { return null; } - const timeShift = selectedColumn.timeShift; + let dateHistogramInterval: null | moment.Duration = null; + const dateHistogramColumn = layer.columnOrder.find( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (dateHistogramColumn && activeData && activeData[layerId] && activeData[layerId]) { + const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); + if (column) { + dateHistogramInterval = search.aggs.parseInterval( + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' + ); + } + } + + function isValueTooSmall(parsedValue: ReturnType) { + return ( + dateHistogramInterval && + parsedValue && + typeof parsedValue === 'object' && + parsedValue.asMilliseconds() < dateHistogramInterval.asMilliseconds() + ); + } + + const parsedLocalValue = localValue && parseTimeShift(localValue); + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid' || localValueTooSmall); function getSelectedOption() { - if (timeShift === '') return []; - const goodPick = timeShiftOptions.filter(({ value }) => value === timeShift); + if (!localValue) return []; + const goodPick = timeShiftOptions.filter(({ value }) => value === localValue); if (goodPick.length > 0) return goodPick; return [ { - value: timeShift, - label: timeShift, + value: localValue, + label: localValue, }, ]; } @@ -147,6 +192,17 @@ export function TimeShift({ label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { defaultMessage: 'Time shift', })} + helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { + defaultMessage: 'Time shift is specified by a number followed by a time unit', + })} + error={ + localValueTooSmall && + i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + }) + } + isInvalid={isLocalValueInvalid} > @@ -161,8 +217,14 @@ export function TimeShift({ options={timeShiftOptions} selectedOptions={getSelectedOption()} singleSelection={{ asPlainText: true }} + isInvalid={isLocalValueInvalid} onCreateOption={(val) => { - updateLayer(setTimeShift(columnId, layer, val)); + const parsedVal = parseTimeShift(val); + if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + updateLayer(setTimeShift(columnId, layer, val)); + } else { + setLocalValue(val); + } }} onChange={(choices) => { if (choices.length === 0) { @@ -171,7 +233,12 @@ export function TimeShift({ } const choice = choices[0].value as string; - updateLayer(setTimeShift(columnId, layer, choice)); + const parsedVal = parseTimeShift(choice); + if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + updateLayer(setTimeShift(columnId, layer, choice)); + } else { + setLocalValue(choice); + } }} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index bd7a270fd7ad8..c045b1a34d3a5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -22,8 +22,10 @@ import { EuiSwitchEvent, EuiTextColor, } from '@elastic/eui'; +import moment from 'moment'; +import { parseTimeShift } from '../../../../../../../src/plugins/data/common'; import { updateColumnParam } from '../layer_helpers'; -import { OperationDefinition } from './index'; +import { OperationDefinition, ParamEditorProps } from './index'; import { FieldBasedIndexPatternColumn } from './column_types'; import { AggFunctionsMapping, @@ -35,6 +37,7 @@ import { import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public'; import { getInvalidFieldMessage, getSafeName } from './helpers'; import { HelpPopover, HelpPopoverButton } from '../../help_popover'; +import { IndexPatternLayer } from '../../types'; const { isValidInterval } = search.aggs; const autoInterval = 'auto'; @@ -48,6 +51,28 @@ export interface DateHistogramIndexPatternColumn extends FieldBasedIndexPatternC }; } +function getMultipleDateHistogramsErrorMessage(layer: IndexPatternLayer, columnId: string) { + const usesTimeShift = Object.values(layer.columns).some( + (col) => col.timeShift && col.timeShift !== '' + ); + if (!usesTimeShift) { + return undefined; + } + const dateHistograms = layer.columnOrder.filter( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (dateHistograms.length < 2) { + return undefined; + } + return i18n.translate('xpack.lens.indexPattern.multipleDateHistogramsError', { + defaultMessage: + '"{dimensionLabel}" is not the only date histogram. When using time shifts, make sure to only use one date histogram.', + values: { + dimensionLabel: layer.columns[columnId].label, + }, + }); +} + export const dateHistogramOperation: OperationDefinition< DateHistogramIndexPatternColumn, 'field' @@ -59,7 +84,13 @@ export const dateHistogramOperation: OperationDefinition< input: 'field', priority: 5, // Highest priority level used getErrorMessage: (layer, columnId, indexPattern) => - getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + [ + ...(getInvalidFieldMessage( + layer.columns[columnId] as FieldBasedIndexPatternColumn, + indexPattern + ) || []), + getMultipleDateHistogramsErrorMessage(layer, columnId) || '', + ].filter(Boolean), getHelpMessage: (props) => , getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => { if ( @@ -149,12 +180,43 @@ export const dateHistogramOperation: OperationDefinition< extended_bounds: JSON.stringify({}), }).toAst(); }, - paramEditor: ({ layer, columnId, currentColumn, updateLayer, dateRange, data, indexPattern }) => { + paramEditor: function ParamEditor({ + layer, + columnId, + currentColumn, + updateLayer, + dateRange, + data, + indexPattern, + activeData, + }: ParamEditorProps) { + const [localValue, setLocalValue] = useState(currentColumn?.params.interval); + useEffect(() => { + setLocalValue(currentColumn?.params.interval); + }, [currentColumn?.params.interval]); + + function isIntervalTooSmall(interval: moment.Duration | null) { + if (!interval) { + return false; + } + const timeShifts = Object.values(layer.columns) + .filter((col) => col.timeShift && col.timeShift !== 'previous') + .map((col) => col.timeShift); + return timeShifts.some((shift) => { + if (!shift) return false; + const parsedShift = parseTimeShift(shift); + return ( + typeof parsedShift === 'object' && + parsedShift.asMilliseconds() < interval.asMilliseconds() + ); + }); + } + const field = currentColumn && indexPattern.getFieldByName(currentColumn.sourceField); const intervalIsRestricted = field!.aggregationRestrictions && field!.aggregationRestrictions.date_histogram; - const interval = parseInterval(currentColumn.params.interval); + const interval = parseInterval(localValue); // We force the interval value to 1 if it's empty, since that is the ES behavior, // and the isValidInterval function doesn't handle the empty case properly. Fixing @@ -172,13 +234,24 @@ export const dateHistogramOperation: OperationDefinition< updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); } - const setInterval = (newInterval: typeof interval) => { + const getIntervalAsDuration = (newInterval: typeof interval) => { const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; + return search.aggs.parseInterval(value); + }; - updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); + const setInterval = (newInterval: typeof interval, storeLocally: boolean = false) => { + const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); + const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; + + if (storeLocally) { + setLocalValue(value); + } else { + updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); + } }; + const currentIntervalTooSmall = isIntervalTooSmall(getIntervalAsDuration(interval)); return ( <> {!intervalIsRestricted && ( @@ -200,6 +273,14 @@ export const dateHistogramOperation: OperationDefinition< })} fullWidth display="rowCompressed" + isInvalid={currentIntervalTooSmall} + error={ + currentIntervalTooSmall && + i18n.translate('xpack.lens.indexPattern.intervalTooSmallError', { + defaultMessage: + 'Interval is larger than configured time shift - make sure to keep date histogram interval smaller than time shifts', + }) + } > {intervalIsRestricted ? ( { - setInterval({ + const newInterval = { ...interval, value: e.target.value, - }); + }; + const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); + setInterval(newInterval, tooSmall); }} /> @@ -237,10 +320,12 @@ export const dateHistogramOperation: OperationDefinition< data-test-subj="lensDateHistogramUnit" value={interval.unit} onChange={(e) => { - setInterval({ + const newInterval = { ...interval, unit: e.target.value, - }); + }; + const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); + setInterval(newInterval, tooSmall); }} isInvalid={!isValid} options={[ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 8fb249501081b..05b746e636edb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -43,6 +43,7 @@ import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; import { RangeIndexPatternColumn, rangeOperation } from './ranges'; +import { IndexPatternDimensionEditorProps } from '../../dimension_panel'; /** * A union type of all available column types. If a column is of an unknown type somewhere @@ -147,6 +148,7 @@ export interface ParamEditorProps { http: HttpSetup; dateRange: DateRange; data: DataPublicPluginStart; + activeData?: IndexPatternDimensionEditorProps['activeData']; } export interface HelpProps { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index d226fe6f2a745..d6d3ea77af4ab 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -50,6 +50,30 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } +function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: string) { + const usesTimeShift = Object.values(layer.columns).some( + (col) => col.timeShift && col.timeShift !== '' + ); + if (!usesTimeShift) { + return undefined; + } + const dateHistogramParent = layer.columnOrder + .slice(0, layer.columnOrder.indexOf(columnId)) + .find((colId) => layer.columns[colId].operationType === 'date_histogram'); + + if (!dateHistogramParent) { + return undefined; + } + return i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { + defaultMessage: + 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', + values: { + dateLabel: layer.columns[dateHistogramParent].label, + dimensionLabel: layer.columns[columnId].label, + }, + }); +} + const DEFAULT_SIZE = 3; const supportedTypes = new Set(['string', 'boolean', 'number', 'ip']); @@ -90,7 +114,13 @@ export const termsOperation: OperationDefinition - getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + [ + ...(getInvalidFieldMessage( + layer.columns[columnId] as FieldBasedIndexPatternColumn, + indexPattern + ) || []), + getInvalidNestingOrderMessage(layer, columnId) || '', + ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 9868359e1cf70..f7989fa8180ff 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -981,6 +981,8 @@ export function updateLayerIndexPattern( * - All columns have complete references * - All column references are valid * - All prerequisites are met + * - If timeshift is used, terms go before date histogram + * - If timeshift is used, only a single date histogram can be used */ export function getErrorMessages( layer: IndexPatternLayer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 19c37da5bf2a9..43e386cff10a3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -61,9 +61,13 @@ export function isColumnInvalid( 'references' in column && Boolean(getReferencesErrors(layer, column, indexPattern).filter(Boolean).length); - return ( - !!operationDefinition.getErrorMessage?.(layer, columnId, indexPattern) || referencesHaveErrors + const operationErrorMessages = operationDefinition.getErrorMessage?.( + layer, + columnId, + indexPattern ); + + return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; } function getReferencesErrors( From 64e208acb131cf9f8b773df2180cf6f196e13977 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 3 May 2021 10:52:33 +0200 Subject: [PATCH 05/31] add better error handling --- .../data/common/search/aggs/agg_configs.ts | 23 ++++++- .../data/common/search/aggs/utils/index.ts | 1 + .../common/search/aggs/utils/shift_error.ts | 20 ++++++ .../editor_frame/config_panel/layer_panel.tsx | 2 + .../editor_frame/editor_frame.tsx | 2 + .../editor_frame/state_management.ts | 8 +++ .../workspace_panel/workspace_panel.tsx | 43 +++++++++++- .../dimension_panel/dimension_editor.tsx | 2 + .../dimension_panel/dimension_panel.tsx | 5 +- .../dimension_panel/time_shift.tsx | 69 ++++++++++++++++--- .../indexpattern_datasource/indexpattern.tsx | 15 ++-- .../operations/definitions/index.ts | 32 +++++++-- .../operations/definitions/terms/index.tsx | 56 +++++++++++---- .../operations/layer_helpers.ts | 36 ++++++++-- .../public/indexpattern_datasource/utils.ts | 13 +++- x-pack/plugins/lens/public/types.ts | 18 ++++- 16 files changed, 293 insertions(+), 52 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/shift_error.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index afd537321e903..24fe5070514db 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,6 +30,7 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; +import { ShiftError } from './utils'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -300,7 +301,27 @@ export class AggConfigs { const timeShiftInterval = config.getTimeShiftInterval(); Object.entries(timeShifts).forEach(([key, shift]) => { if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { - throw new Error('All time shifts need to be larger than underlying date interval'); + const aggIds = this.getAll() + .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) + .map((agg) => agg.id); + throw new ShiftError( + `All time shifts need to be larger than underlying date interval of ${timeShiftInterval.humanize()}`, + aggIds, + 'tooSmall' + ); + } + if ( + timeShiftInterval && + !Number.isInteger(shift.asMilliseconds() / timeShiftInterval.asMilliseconds()) + ) { + const aggIds = this.getAll() + .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) + .map((agg) => agg.id); + throw new ShiftError( + `All time shifts need to be a multiple of the underlying interval of ${timeShiftInterval.humanize()}`, + aggIds, + 'notAMultiple' + ); } filters[key] = { range: { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index 1f2d0a308f0dd..8649686df3736 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -16,3 +16,4 @@ export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; export * from './parse_time_shift'; +export * from './shift_error'; diff --git a/src/plugins/data/common/search/aggs/utils/shift_error.ts b/src/plugins/data/common/search/aggs/utils/shift_error.ts new file mode 100644 index 0000000000000..49f4f3714f5e9 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/shift_error.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Configuration error of time shift + */ +export class ShiftError extends Error { + constructor( + message: string, + public aggIds: string[], + public reason: 'tooSmall' | 'notAMultiple' + ) { + super(message); + } +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index cf3c9099d4b0d..3b6e2b2a3eb02 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -357,6 +357,7 @@ export function LayerPanel( columnId: accessorConfig.columnId, groupId: group.groupId, filterOperations: group.filterOperations, + runtimeError: props.framePublicAPI.runtimeError, }} /> @@ -433,6 +434,7 @@ export function LayerPanel( hideGrouping: activeGroup.hideGrouping, filterOperations: activeGroup.filterOperations, dimensionGroups: groups, + runtimeError: props.framePublicAPI.runtimeError, setState: ( newState: unknown, { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 362787ea91c4f..642ef76da4f36 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -110,6 +110,7 @@ export function EditorFrame(props: EditorFrameProps) { const framePublicAPI: FramePublicAPI = { datasourceLayers, activeData: state.activeData, + runtimeError: state.lastRuntimeError, dateRange: props.dateRange, query: props.query, filters: props.filters, @@ -366,6 +367,7 @@ export function EditorFrame(props: EditorFrameProps) { plugins={props.plugins} visualizeTriggerFieldContext={visualizeTriggerFieldContext} getSuggestionForField={getSuggestionForField} + hasRuntimeError={Boolean(state.lastRuntimeError)} /> ) } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index 53aba0d6f3f6c..d49ee4cec46f3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,6 +24,7 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; + lastRuntimeError?: Error; } export type Action = @@ -31,6 +32,10 @@ export type Action = type: 'RESET'; state: EditorFrameState; } + | { + type: 'RUNTIME_ERROR'; + error: Error; + } | { type: 'UPDATE_TITLE'; title: string; @@ -144,11 +149,14 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta return action.state; case 'UPDATE_TITLE': return { ...state, title: action.title }; + case 'RUNTIME_ERROR': + return { ...state, lastRuntimeError: action.error }; case 'UPDATE_STATE': return action.updater(state); case 'UPDATE_ACTIVE_DATA': return { ...state, + lastRuntimeError: undefined, activeData: { ...action.tables }, }; case 'UPDATE_LAYER': diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index a31146e500434..389475d07d831 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -18,6 +18,7 @@ import { EuiButtonEmpty, EuiLink, EuiPageContentBody, + EuiButton, } from '@elastic/eui'; import { CoreStart, ApplicationStart } from 'kibana/public'; import { @@ -74,12 +75,17 @@ export interface WorkspacePanelProps { core: CoreStart; plugins: { uiActions?: UiActionsStart; data: DataPublicPluginStart }; title?: string; + hasRuntimeError?: boolean; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; } interface WorkspaceState { - expressionBuildError?: Array<{ shortMessage: string; longMessage: string }>; + expressionBuildError?: Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: unknown }; + }>; expandError: boolean; } @@ -127,6 +133,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, + hasRuntimeError, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -334,6 +341,8 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ localState={{ ...localState, configurationValidationError, missingRefsErrors }} ExpressionRendererComponent={ExpressionRendererComponent} application={core.application} + hasRuntimeError={hasRuntimeError} + activeDatasourceId={activeDatasourceId} /> ); }; @@ -377,6 +386,8 @@ export const VisualizationWrapper = ({ ExpressionRendererComponent, dispatch, application, + hasRuntimeError, + activeDatasourceId, }: { expression: string | null | undefined; framePublicAPI: FramePublicAPI; @@ -385,11 +396,17 @@ export const VisualizationWrapper = ({ dispatch: (action: Action) => void; setLocalState: (dispatch: (prevState: WorkspaceState) => WorkspaceState) => void; localState: WorkspaceState & { - configurationValidationError?: Array<{ shortMessage: string; longMessage: string }>; + configurationValidationError?: Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: unknown }; + }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; ExpressionRendererComponent: ReactExpressionRendererType; application: ApplicationStart; + hasRuntimeError?: boolean; + activeDatasourceId: string | null; }) => { const context: ExecutionContextSearch = useMemo( () => ({ @@ -467,6 +484,19 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].longMessage}

+ {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( + { + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + datasourceId: activeDatasourceId, + updater: localState.configurationValidationError?.[0].fixAction?.newState, + }); + }} + > + {localState.configurationValidationError[0].fixAction.label} + + )} {showExtraErrors} @@ -526,6 +556,7 @@ export const VisualizationWrapper = ({ } if (localState.expressionBuildError?.length) { + const firstError = localState.expressionBuildError[0]; return ( @@ -539,7 +570,7 @@ export const VisualizationWrapper = ({ />

-

{localState.expressionBuildError[0].longMessage}

+

{firstError.longMessage}

} iconColor="danger" @@ -562,6 +593,12 @@ export const VisualizationWrapper = ({ onData$={onData$} renderMode="edit" renderError={(errorMessage?: string | null, error?: ExpressionRenderError | null) => { + if (!hasRuntimeError && error?.original) { + dispatch({ + type: 'RUNTIME_ERROR', + error: error.original, + }); + } const errorsFromRequest = getOriginalRequestErrorMessages(error); const visibleErrorMessages = errorsFromRequest.length ? errorsFromRequest diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 73fdb6fe99ec8..c5f653fa22cfa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -121,6 +121,7 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, + runtimeError, } = props; const services = { data: props.data, @@ -546,6 +547,7 @@ export function DimensionEditor(props: DimensionEditorProps) { isFocused={timeShiftedFocused} activeData={props.activeData} layerId={layerId} + runtimeError={runtimeError} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx index 3965f992805b5..ccf5f53d6c9d5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx @@ -44,14 +44,15 @@ function wrapOnDot(str?: string) { export const IndexPatternDimensionTriggerComponent = function IndexPatternDimensionTrigger( props: IndexPatternDimensionTriggerProps ) { + const runtimeError = props.runtimeError; const layerId = props.layerId; const layer = props.state.layers[layerId]; const currentIndexPattern = props.state.indexPatterns[layer.indexPatternId]; const { columnId, uniqueLabel } = props; const currentColumnHasErrors = useMemo( - () => isColumnInvalid(layer, columnId, currentIndexPattern), - [layer, columnId, currentIndexPattern] + () => isColumnInvalid(layer, columnId, currentIndexPattern, runtimeError), + [layer, columnId, currentIndexPattern, runtimeError] ); const selectedColumn: IndexPatternColumn | null = layer.columns[props.columnId] ?? null; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 8db0af5a71f0e..b9052e4b9e40f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; -import { parseTimeShift } from '../../../../../../src/plugins/data/common'; +import { parseTimeShift, ShiftError } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; @@ -118,6 +118,7 @@ export function TimeShift({ isFocused, activeData, layerId, + runtimeError, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -127,6 +128,7 @@ export function TimeShift({ isFocused: boolean; activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; + runtimeError?: Error | undefined; }) { const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { @@ -159,9 +161,35 @@ export function TimeShift({ ); } + function isValueNotMultiple(parsedValue: ReturnType) { + return ( + dateHistogramInterval && + parsedValue && + typeof parsedValue === 'object' && + !Number.isInteger(parsedValue.asMilliseconds() / dateHistogramInterval.asMilliseconds()) + ); + } + + function getRuntimeErrorReason() { + if (!(runtimeError instanceof ShiftError)) { + return undefined; + } + if (!runtimeError.aggIds.includes(columnId)) { + return undefined; + } + return runtimeError.reason; + } + const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); - const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid' || localValueTooSmall); + const localValueTooSmall = + (parsedLocalValue && isValueTooSmall(parsedLocalValue)) || + getRuntimeErrorReason() === 'tooSmall'; + const localValueNotMultiple = + (parsedLocalValue && isValueNotMultiple(parsedLocalValue)) || + getRuntimeErrorReason() === 'notAMultiple'; + const isLocalValueInvalid = Boolean( + parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple + ); function getSelectedOption() { if (!localValue) return []; @@ -196,11 +224,17 @@ export function TimeShift({ defaultMessage: 'Time shift is specified by a number followed by a time unit', })} error={ - localValueTooSmall && - i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { - defaultMessage: - 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', - }) + localValueTooSmall + ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + }) + : localValueNotMultiple + ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { + defaultMessage: + 'Time shift has to be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + }) + : undefined } isInvalid={isLocalValueInvalid} > @@ -214,13 +248,22 @@ export function TimeShift({ placeholder={i18n.translate('xpack.lens.indexPattern.timeShiftPlaceholder', { defaultMessage: 'Time shift (e.g. 1d)', })} - options={timeShiftOptions} + options={timeShiftOptions.filter(({ value }) => { + const parsedValue = parseTimeShift(value); + return ( + parsedValue && !isValueTooSmall(parsedValue) && !isValueNotMultiple(parsedValue) + ); + })} selectedOptions={getSelectedOption()} singleSelection={{ asPlainText: true }} isInvalid={isLocalValueInvalid} onCreateOption={(val) => { const parsedVal = parseTimeShift(val); - if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + if ( + parsedVal !== 'invalid' && + !isValueTooSmall(parsedVal) && + !isValueNotMultiple(parsedVal) + ) { updateLayer(setTimeShift(columnId, layer, val)); } else { setLocalValue(val); @@ -234,7 +277,11 @@ export function TimeShift({ const choice = choices[0].value as string; const parsedVal = parseTimeShift(choice); - if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + if ( + parsedVal !== 'invalid' && + !isValueTooSmall(parsedVal) && + !isValueNotMultiple(parsedVal) + ) { updateLayer(setTimeShift(columnId, layer, choice)); } else { setLocalValue(choice); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 81eb46e816715..dfb0d74dd7cb5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -374,13 +374,14 @@ export function getIndexPatternDatasource({ } // Forward the indexpattern as well, as it is required by some operationType checks - const layerErrors = Object.values(state.layers).map((layer) => - (getErrorMessages(layer, state.indexPatterns[layer.indexPatternId]) ?? []).map( - (message) => ({ - shortMessage: '', // Not displayed currently - longMessage: message, - }) - ) + const layerErrors = Object.entries(state.layers).map(([layerId, layer]) => + ( + getErrorMessages(layer, state.indexPatterns[layer.indexPatternId], state, layerId) ?? [] + ).map((message) => ({ + shortMessage: '', // Not displayed currently + longMessage: typeof message === 'string' ? message : message.message, + fixAction: typeof message === 'object' ? message.fixAction : undefined, + })) ); // Single layer case, no need to explain more diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 05b746e636edb..ace51624c5c71 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -38,7 +38,12 @@ import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; +import { + IndexPattern, + IndexPatternField, + IndexPatternLayer, + IndexPatternPrivateState, +} from '../../types'; import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; @@ -222,8 +227,16 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern - ) => string[] | undefined; + indexPattern: IndexPattern, + // TODO - remove these again and remap the state on a higher level + state?: IndexPatternPrivateState, + layerId?: string + ) => + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined; /* * Flag whether this operation can be scaled by time unit if a date histogram is available. @@ -320,11 +333,18 @@ interface FieldBasedOperationDefinition { * - Requires a date histogram operation somewhere before it in order * - Missing references */ - getErrorMessage: ( + getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern - ) => string[] | undefined; + indexPattern: IndexPattern, + state?: IndexPatternPrivateState, + layerId?: string + ) => + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined; } export interface RequiredReference { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index d6d3ea77af4ab..77c76398017c4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -24,7 +24,7 @@ import { OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; -import type { IndexPatternLayer } from '../../../types'; +import type { IndexPatternLayer, IndexPatternPrivateState } from '../../../types'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -50,7 +50,12 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } -function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: string) { +function getInvalidNestingOrderMessage( + layer: IndexPatternLayer, + columnId: string, + state?: IndexPatternPrivateState, + layerId?: string +) { const usesTimeShift = Object.values(layer.columns).some( (col) => col.timeShift && col.timeShift !== '' ); @@ -64,14 +69,41 @@ function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: strin if (!dateHistogramParent) { return undefined; } - return i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { - defaultMessage: - 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', - values: { - dateLabel: layer.columns[dateHistogramParent].label, - dimensionLabel: layer.columns[columnId].label, - }, - }); + return { + message: i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { + defaultMessage: + 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', + values: { + dateLabel: layer.columns[dateHistogramParent].label, + dimensionLabel: layer.columns[columnId].label, + }, + }), + fixAction: + state && layerId + ? { + label: i18n.translate( + 'xpack.lens.indexPattern.termsInShiftedDateHistogramFixActionLabel', + { + defaultMessage: 'Reorder dimensions', + } + ), + newState: { + ...state, + layers: { + ...state.layers, + [layerId]: { + ...layer, + columns: { + ...layer.columns, + [dateHistogramParent]: layer.columns[columnId], + [columnId]: layer.columns[dateHistogramParent], + }, + }, + }, + }, + } + : undefined, + }; } const DEFAULT_SIZE = 3; @@ -113,13 +145,13 @@ export const termsOperation: OperationDefinition + getErrorMessage: (layer, columnId, indexPattern, state, layerId) => [ ...(getInvalidFieldMessage( layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern ) || []), - getInvalidNestingOrderMessage(layer, columnId) || '', + getInvalidNestingOrderMessage(layer, columnId, state, layerId) || '', ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index f7989fa8180ff..8ef6831890cbd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -14,7 +14,12 @@ import { IndexPatternColumn, RequiredReference, } from './definitions'; -import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types'; +import type { + IndexPattern, + IndexPatternField, + IndexPatternLayer, + IndexPatternPrivateState, +} from '../types'; import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; @@ -986,17 +991,36 @@ export function updateLayerIndexPattern( */ export function getErrorMessages( layer: IndexPatternLayer, - indexPattern: IndexPattern -): string[] | undefined { - const errors: string[] = Object.entries(layer.columns) + indexPattern: IndexPattern, + state: IndexPatternPrivateState, + layerId: string +): + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined { + const errors: Array< + string | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern); + return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); } }) // remove the undefined values - .filter((v: string | undefined): v is string => v != null); + .filter( + ( + v: + | string + | undefined + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + ): v is + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } => + v != null + ); return errors.length ? errors : undefined; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 43e386cff10a3..3734f18eb8a59 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { ShiftError } from '../../../../../src/plugins/data/common'; import { DataType } from '../types'; import { IndexPattern, IndexPatternLayer, DraggedField } from './types'; import type { @@ -49,7 +50,8 @@ export function isDraggedField(fieldCandidate: unknown): fieldCandidate is Dragg export function isColumnInvalid( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + runtimeError?: Error ) { const column: IndexPatternColumn | undefined = layer.columns[columnId]; if (!column) return; @@ -67,7 +69,14 @@ export function isColumnInvalid( indexPattern ); - return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; + const hasTimeShiftError = + runtimeError instanceof ShiftError && runtimeError.aggIds.includes(columnId); + + return ( + (operationErrorMessages && operationErrorMessages.length > 0) || + referencesHaveErrors || + hasTimeShiftError + ); } function getReferencesErrors( diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 94b4433a82551..2a7e261f66871 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -236,7 +236,13 @@ export interface Datasource { getErrorMessages: ( state: T, layersGroups?: Record - ) => Array<{ shortMessage: string; longMessage: string }> | undefined; + ) => + | Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: T }; + }> + | undefined; /** * uniqueLabels of dimensions exposed for aria-labels of dragged dimensions */ @@ -289,6 +295,7 @@ export type DatasourceDimensionProps = SharedDimensionProps & { onRemove?: (accessor: string) => void; state: T; activeData?: Record; + runtimeError?: Error; }; // The only way a visualization has to restrict the query building @@ -310,6 +317,7 @@ export interface DatasourceLayerPanelProps { state: T; setState: StateSetter; activeData?: Record; + runtimeError?: Error; } export interface DraggedOperation extends DraggingIdentifier { @@ -510,6 +518,7 @@ export interface FramePublicAPI { * If accessing, make sure to check whether expected columns actually exist. */ activeData?: Record; + runtimeError?: Error; dateRange: DateRange; query: Query; @@ -682,7 +691,12 @@ export interface Visualization { getErrorMessages: ( state: T, datasourceLayers?: Record - ) => Array<{ shortMessage: string; longMessage: string }> | undefined; + ) => + | Array<{ + shortMessage: string; + longMessage: string; + }> + | undefined; /** * The frame calls this function to display warnings about visualization From 5b389ce92131080d2516347332684bcdfb67aefd Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 3 May 2021 15:40:08 +0200 Subject: [PATCH 06/31] fix broken metric time shift --- .../data/common/search/aggs/metrics/metric_agg_type.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index da71383dd564c..9186ba8f70cfc 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -81,7 +81,9 @@ export class MetricAggType - aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg; + aggs.hasTimeShifts() && + aggs.byType(AggGroupNames.Metrics)[0] === agg && + !aggs.byType(AggGroupNames.Buckets).some((bucketAgg) => bucketAgg.splitForTimeShift(aggs)); } } From 20940a7a26412e87a1349704a85beaf8dd97a5f9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:00:48 +0200 Subject: [PATCH 07/31] experimental fix action --- .../workspace_panel/workspace_panel.tsx | 9 ++- .../dimension_panel/time_shift.tsx | 44 ++++------ .../indexpattern_datasource/indexpattern.tsx | 8 +- .../operations/definitions/index.ts | 16 +++- .../operations/definitions/terms/index.tsx | 81 ++++++++++++++++++- .../operations/layer_helpers.ts | 46 +++++++++-- .../public/indexpattern_datasource/utils.ts | 10 +-- x-pack/plugins/lens/public/types.ts | 2 +- 8 files changed, 163 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 389475d07d831..d563cd05ffd56 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -84,7 +84,7 @@ interface WorkspaceState { expressionBuildError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: unknown }; + fixAction?: { label: string; newState: () => Promise }; }>; expandError: boolean; } @@ -399,7 +399,7 @@ export const VisualizationWrapper = ({ configurationValidationError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: unknown }; + fixAction?: { label: string; newState: () => Promise }; }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; @@ -486,11 +486,12 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { + onClick={async () => { + const newState = await localState.configurationValidationError?.[0].fixAction?.newState(); dispatch({ type: 'UPDATE_DATASOURCE_STATE', datasourceId: activeDatasourceId, - updater: localState.configurationValidationError?.[0].fixAction?.newState, + updater: newState, }); }} > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index b9052e4b9e40f..d4e58ee77b73b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -227,12 +227,12 @@ export function TimeShift({ localValueTooSmall ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { defaultMessage: - 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', }) : localValueNotMultiple ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { defaultMessage: - 'Time shift has to be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', }) : undefined } @@ -259,11 +259,7 @@ export function TimeShift({ isInvalid={isLocalValueInvalid} onCreateOption={(val) => { const parsedVal = parseTimeShift(val); - if ( - parsedVal !== 'invalid' && - !isValueTooSmall(parsedVal) && - !isValueNotMultiple(parsedVal) - ) { + if (parsedVal !== 'invalid') { updateLayer(setTimeShift(columnId, layer, val)); } else { setLocalValue(val); @@ -277,11 +273,7 @@ export function TimeShift({ const choice = choices[0].value as string; const parsedVal = parseTimeShift(choice); - if ( - parsedVal !== 'invalid' && - !isValueTooSmall(parsedVal) && - !isValueNotMultiple(parsedVal) - ) { + if (parsedVal !== 'invalid') { updateLayer(setTimeShift(columnId, layer, choice)); } else { setLocalValue(choice); @@ -289,21 +281,19 @@ export function TimeShift({ }} /> - {selectedOperation.timeScalingMode === 'optional' && ( - - { - updateLayer(setTimeShift(columnId, layer, undefined)); - }} - iconType="cross" - /> - - )} + + { + updateLayer(setTimeShift(columnId, layer, undefined)); + }} + iconType="cross" + /> + diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index dfb0d74dd7cb5..8ed6c2c3126f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -376,7 +376,13 @@ export function getIndexPatternDatasource({ // Forward the indexpattern as well, as it is required by some operationType checks const layerErrors = Object.entries(state.layers).map(([layerId, layer]) => ( - getErrorMessages(layer, state.indexPatterns[layer.indexPatternId], state, layerId) ?? [] + getErrorMessages( + layer, + state.indexPatterns[layer.indexPatternId], + state, + layerId, + core.http + ) ?? [] ).map((message) => ({ shortMessage: '', // Not displayed currently longMessage: typeof message === 'string' ? message : message.message, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index ace51624c5c71..c89ada8c7a670 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -234,7 +234,13 @@ interface BaseOperationDefinitionProps { ) => | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined; @@ -342,7 +348,13 @@ interface FieldBasedOperationDefinition { ) => | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 77c76398017c4..8b0822a93bc30 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -16,11 +16,14 @@ import { EuiAccordion, EuiIconTip, } from '@elastic/eui'; +import { uniq } from 'lodash'; +import { HttpSetup } from 'kibana/public'; +import { FieldStatsResponse } from '../../../../../common'; import { AggFunctionsMapping } from '../../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../../src/plugins/expressions/public'; import { updateColumnParam, isReferenced } from '../../layer_helpers'; import { DataType } from '../../../../types'; -import { OperationDefinition } from '../index'; +import { FiltersIndexPatternColumn, OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; @@ -50,6 +53,77 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } +function getDisallowedTermsMessage( + layer: IndexPatternLayer, + columnId: string, + state?: IndexPatternPrivateState, + layerId?: string +) { + const hasMultipleShifts = + uniq(Object.values(layer.columns).map((col) => col.timeShift !== '')).length > 1; + if (!hasMultipleShifts) { + return undefined; + } + const dateHistogramParent = layer.columnOrder + .slice(0, layer.columnOrder.indexOf(columnId)) + .find((colId) => layer.columns[colId].operationType === 'date_histogram'); + + if (!dateHistogramParent) { + return undefined; + } + return { + message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { + defaultMessage: + "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + }), + fixAction: + state && layerId + ? { + label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { + defaultMessage: 'Pin top values', + }), + newState: async (http: HttpSetup) => { + const indexPattern = state.indexPatterns[layer.indexPatternId]; + const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; + const response: FieldStatsResponse = await http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + }), + } + ); + return { + ...state, + layers: { + ...state.layers, + [layerId]: { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + ...layer.columns[columnId], + operationType: 'filters', + params: { + filters: response.topValues?.buckets.map(({ key }) => ({ + input: { + query: `${fieldName}: "${key}"`, + language: 'kuery', + }, + label: '', + })), + }, + } as FiltersIndexPatternColumn, + }, + }, + }, + }; + }, + } + : undefined, + }; +} + function getInvalidNestingOrderMessage( layer: IndexPatternLayer, columnId: string, @@ -87,7 +161,7 @@ function getInvalidNestingOrderMessage( defaultMessage: 'Reorder dimensions', } ), - newState: { + newState: async () => ({ ...state, layers: { ...state.layers, @@ -100,7 +174,7 @@ function getInvalidNestingOrderMessage( }, }, }, - }, + }), } : undefined, }; @@ -152,6 +226,7 @@ export const termsOperation: OperationDefinition { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 8ef6831890cbd..8544bc7c2f108 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -6,6 +6,7 @@ */ import _, { partition } from 'lodash'; +import { HttpSetup } from 'kibana/public'; import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types'; import { operationDefinitionMap, @@ -993,15 +994,29 @@ export function getErrorMessages( layer: IndexPatternLayer, indexPattern: IndexPattern, state: IndexPatternPrivateState, - layerId: string + layerId: string, + http: HttpSetup ): | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined { const errors: Array< - string | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | string + | { + message: string; + fixAction?: { + label: string; + newState: () => Promise; + }; + } > = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; @@ -1009,17 +1024,36 @@ export function getErrorMessages( return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); } }) + .map((errorMessage) => { + if (typeof errorMessage !== 'object') { + return errorMessage; + } + return { + ...errorMessage, + fixAction: errorMessage.fixAction + ? { + ...errorMessage.fixAction, + newState: () => errorMessage.fixAction!.newState(http), + } + : undefined, + }; + }) // remove the undefined values .filter( ( v: | string | undefined - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { label: string; newState: () => Promise }; + } ): v is | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } => - v != null + | { + message: string; + fixAction?: { label: string; newState: () => Promise }; + } => v != null ); return errors.length ? errors : undefined; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 3734f18eb8a59..179f8b2f1c5f2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { ShiftError } from '../../../../../src/plugins/data/common'; import { DataType } from '../types'; import { IndexPattern, IndexPatternLayer, DraggedField } from './types'; import type { @@ -69,14 +68,7 @@ export function isColumnInvalid( indexPattern ); - const hasTimeShiftError = - runtimeError instanceof ShiftError && runtimeError.aggIds.includes(columnId); - - return ( - (operationErrorMessages && operationErrorMessages.length > 0) || - referencesHaveErrors || - hasTimeShiftError - ); + return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; } function getReferencesErrors( diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 2a7e261f66871..01ac323d88b33 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -240,7 +240,7 @@ export interface Datasource { | Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: T }; + fixAction?: { label: string; newState: () => Promise }; }> | undefined; /** From 5603bb922f45394ad94f3d9bde3231f4ccb3baf5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:33:29 +0200 Subject: [PATCH 08/31] cleanup --- .../data/common/search/aggs/agg_configs.ts | 25 --------- .../data/common/search/aggs/utils/index.ts | 1 - .../common/search/aggs/utils/shift_error.ts | 20 -------- .../workspace_panel/workspace_panel.tsx | 8 +-- .../indexpattern_datasource/indexpattern.tsx | 2 +- .../operations/definitions/index.ts | 14 +++-- .../operations/definitions/terms/index.tsx | 51 ++++++++++++------- .../operations/layer_helpers.ts | 51 ++++++++----------- x-pack/plugins/lens/public/types.ts | 6 ++- 9 files changed, 75 insertions(+), 103 deletions(-) delete mode 100644 src/plugins/data/common/search/aggs/utils/shift_error.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 24fe5070514db..8dbde8b982586 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,7 +30,6 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; -import { ShiftError } from './utils'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -298,31 +297,7 @@ export class AggConfigs { }, }, }; - const timeShiftInterval = config.getTimeShiftInterval(); Object.entries(timeShifts).forEach(([key, shift]) => { - if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { - const aggIds = this.getAll() - .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) - .map((agg) => agg.id); - throw new ShiftError( - `All time shifts need to be larger than underlying date interval of ${timeShiftInterval.humanize()}`, - aggIds, - 'tooSmall' - ); - } - if ( - timeShiftInterval && - !Number.isInteger(shift.asMilliseconds() / timeShiftInterval.asMilliseconds()) - ) { - const aggIds = this.getAll() - .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) - .map((agg) => agg.id); - throw new ShiftError( - `All time shifts need to be a multiple of the underlying interval of ${timeShiftInterval.humanize()}`, - aggIds, - 'notAMultiple' - ); - } filters[key] = { range: { [timeField]: { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index 8649686df3736..1f2d0a308f0dd 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -16,4 +16,3 @@ export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; export * from './parse_time_shift'; -export * from './shift_error'; diff --git a/src/plugins/data/common/search/aggs/utils/shift_error.ts b/src/plugins/data/common/search/aggs/utils/shift_error.ts deleted file mode 100644 index 49f4f3714f5e9..0000000000000 --- a/src/plugins/data/common/search/aggs/utils/shift_error.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/** - * Configuration error of time shift - */ -export class ShiftError extends Error { - constructor( - message: string, - public aggIds: string[], - public reason: 'tooSmall' | 'notAMultiple' - ) { - super(message); - } -} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index d563cd05ffd56..830fff3acddde 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -84,7 +84,7 @@ interface WorkspaceState { expressionBuildError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: () => Promise }; + fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise }; }>; expandError: boolean; } @@ -399,7 +399,7 @@ export const VisualizationWrapper = ({ configurationValidationError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: () => Promise }; + fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise }; }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; @@ -487,7 +487,9 @@ export const VisualizationWrapper = ({ {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { - const newState = await localState.configurationValidationError?.[0].fixAction?.newState(); + const newState = await localState.configurationValidationError?.[0].fixAction?.newState( + framePublicAPI + ); dispatch({ type: 'UPDATE_DATASOURCE_STATE', datasourceId: activeDatasourceId, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 8ed6c2c3126f4..01dae7df64c83 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -381,7 +381,7 @@ export function getIndexPatternDatasource({ state.indexPatterns[layer.indexPatternId], state, layerId, - core.http + core ) ?? [] ).map((message) => ({ shortMessage: '', // Not displayed currently diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index c89ada8c7a670..9421433eaf780 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreStart } from 'kibana/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { termsOperation, TermsIndexPatternColumn } from './terms'; import { filtersOperation, FiltersIndexPatternColumn } from './filters'; @@ -36,7 +36,7 @@ import { } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; -import { OperationMetadata } from '../../../types'; +import { FramePublicAPI, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, @@ -238,7 +238,10 @@ interface BaseOperationDefinitionProps { message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI + ) => Promise; }; } > @@ -352,7 +355,10 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI + ) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 8b0822a93bc30..9f42ba4daaf47 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -17,13 +17,17 @@ import { EuiIconTip, } from '@elastic/eui'; import { uniq } from 'lodash'; -import { HttpSetup } from 'kibana/public'; +import { CoreStart } from 'kibana/public'; import { FieldStatsResponse } from '../../../../../common'; -import { AggFunctionsMapping } from '../../../../../../../../src/plugins/data/public'; +import { + AggFunctionsMapping, + esQuery, + IIndexPattern, +} from '../../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../../src/plugins/expressions/public'; import { updateColumnParam, isReferenced } from '../../layer_helpers'; -import { DataType } from '../../../../types'; -import { FiltersIndexPatternColumn, OperationDefinition } from '../index'; +import { DataType, FramePublicAPI } from '../../../../types'; +import { FiltersIndexPatternColumn, OperationDefinition, operationDefinitionMap } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; @@ -60,17 +64,14 @@ function getDisallowedTermsMessage( layerId?: string ) { const hasMultipleShifts = - uniq(Object.values(layer.columns).map((col) => col.timeShift !== '')).length > 1; + uniq( + Object.values(layer.columns) + .filter((col) => operationDefinitionMap[col.operationType].shiftable) + .map((col) => col.timeShift || '') + ).length > 1; if (!hasMultipleShifts) { return undefined; } - const dateHistogramParent = layer.columnOrder - .slice(0, layer.columnOrder.indexOf(columnId)) - .find((colId) => layer.columns[colId].operationType === 'date_histogram'); - - if (!dateHistogramParent) { - return undefined; - } return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: @@ -80,16 +81,24 @@ function getDisallowedTermsMessage( state && layerId ? { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin top values', + defaultMessage: 'Pin current top values', }), - newState: async (http: HttpSetup) => { + newState: async (core: CoreStart, frame: FramePublicAPI) => { const indexPattern = state.indexPatterns[layer.indexPatternId]; const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await http.post( + const response: FieldStatsResponse = await core.http.post( `/api/lens/index_stats/${indexPattern.id}/field`, { body: JSON.stringify({ fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, }), } ); @@ -102,7 +111,15 @@ function getDisallowedTermsMessage( columns: { ...layer.columns, [columnId]: { - ...layer.columns[columnId], + label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { + defaultMessage: 'Pinned top values of {field}', + values: { + field: fieldName, + }, + }), + customLabel: true, + isBucketed: layer.columns[columnId].isBucketed, + dataType: 'string', operationType: 'filters', params: { filters: response.topValues?.buckets.map(({ key }) => ({ @@ -110,7 +127,7 @@ function getDisallowedTermsMessage( query: `${fieldName}: "${key}"`, language: 'kuery', }, - label: '', + label: String(key), })), }, } as FiltersIndexPatternColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 787139c75a328..ffac563eeeea8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -6,8 +6,12 @@ */ import _, { partition } from 'lodash'; -import { HttpSetup } from 'kibana/public'; -import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types'; +import { CoreStart } from 'kibana/public'; +import type { + FramePublicAPI, + OperationMetadata, + VisualizationDimensionGroupConfig, +} from '../../types'; import { operationDefinitionMap, operationDefinitions, @@ -1075,7 +1079,7 @@ export function getErrorMessages( indexPattern: IndexPattern, state: IndexPatternPrivateState, layerId: string, - http: HttpSetup + core: CoreStart ): | Array< | string @@ -1083,21 +1087,12 @@ export function getErrorMessages( message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: (frame: FramePublicAPI) => Promise; }; } > | undefined { - const errors: Array< - | string - | { - message: string; - fixAction?: { - label: string; - newState: () => Promise; - }; - } - > = Object.entries(layer.columns) + const errors = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { @@ -1113,28 +1108,22 @@ export function getErrorMessages( fixAction: errorMessage.fixAction ? { ...errorMessage.fixAction, - newState: () => errorMessage.fixAction!.newState(http), + newState: (frame: FramePublicAPI) => errorMessage.fixAction!.newState(core, frame), } : undefined, }; }) // remove the undefined values - .filter( - ( - v: - | string - | undefined - | { - message: string; - fixAction?: { label: string; newState: () => Promise }; - } - ): v is - | string - | { - message: string; - fixAction?: { label: string; newState: () => Promise }; - } => v != null - ); + .filter((v) => v != null) as Array< + | string + | { + message: string; + fixAction?: { + label: string; + newState: (framePublicAPI: FramePublicAPI) => Promise; + }; + } + >; return errors.length ? errors : undefined; } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 01ac323d88b33..5516f16e0bbae 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -235,7 +235,11 @@ export interface Datasource { getPublicAPI: (props: PublicAPIProps) => DatasourcePublicAPI; getErrorMessages: ( state: T, - layersGroups?: Record + layersGroups?: Record, + dateRange?: { + fromDate: string; + toDate: string; + } ) => | Array<{ shortMessage: string; From 8f6b95c369895f98dc198523d647da0641a6b823 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:54:55 +0200 Subject: [PATCH 09/31] remove current runtime error thingy --- .../workspace_panel/workspace_panel.tsx | 11 ---------- .../dimension_panel/dimension_editor.tsx | 2 -- .../dimension_panel/dimension_panel.tsx | 5 ++--- .../dimension_panel/time_shift.tsx | 22 +++---------------- .../public/indexpattern_datasource/utils.ts | 3 +-- x-pack/plugins/lens/public/types.ts | 3 --- 6 files changed, 6 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 830fff3acddde..585ac73bfea9b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -75,7 +75,6 @@ export interface WorkspacePanelProps { core: CoreStart; plugins: { uiActions?: UiActionsStart; data: DataPublicPluginStart }; title?: string; - hasRuntimeError?: boolean; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; } @@ -133,7 +132,6 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, - hasRuntimeError, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -341,7 +339,6 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ localState={{ ...localState, configurationValidationError, missingRefsErrors }} ExpressionRendererComponent={ExpressionRendererComponent} application={core.application} - hasRuntimeError={hasRuntimeError} activeDatasourceId={activeDatasourceId} /> ); @@ -386,7 +383,6 @@ export const VisualizationWrapper = ({ ExpressionRendererComponent, dispatch, application, - hasRuntimeError, activeDatasourceId, }: { expression: string | null | undefined; @@ -405,7 +401,6 @@ export const VisualizationWrapper = ({ }; ExpressionRendererComponent: ReactExpressionRendererType; application: ApplicationStart; - hasRuntimeError?: boolean; activeDatasourceId: string | null; }) => { const context: ExecutionContextSearch = useMemo( @@ -596,12 +591,6 @@ export const VisualizationWrapper = ({ onData$={onData$} renderMode="edit" renderError={(errorMessage?: string | null, error?: ExpressionRenderError | null) => { - if (!hasRuntimeError && error?.original) { - dispatch({ - type: 'RUNTIME_ERROR', - error: error.original, - }); - } const errorsFromRequest = getOriginalRequestErrorMessages(error); const visibleErrorMessages = errorsFromRequest.length ? errorsFromRequest diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index e51c4f2c02f1b..a469924fac054 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -120,7 +120,6 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, - runtimeError, } = props; const services = { data: props.data, @@ -537,7 +536,6 @@ export function DimensionEditor(props: DimensionEditorProps) { isFocused={timeShiftedFocused} activeData={props.activeData} layerId={layerId} - runtimeError={runtimeError} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx index ccf5f53d6c9d5..3965f992805b5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx @@ -44,15 +44,14 @@ function wrapOnDot(str?: string) { export const IndexPatternDimensionTriggerComponent = function IndexPatternDimensionTrigger( props: IndexPatternDimensionTriggerProps ) { - const runtimeError = props.runtimeError; const layerId = props.layerId; const layer = props.state.layers[layerId]; const currentIndexPattern = props.state.indexPatterns[layer.indexPatternId]; const { columnId, uniqueLabel } = props; const currentColumnHasErrors = useMemo( - () => isColumnInvalid(layer, columnId, currentIndexPattern, runtimeError), - [layer, columnId, currentIndexPattern, runtimeError] + () => isColumnInvalid(layer, columnId, currentIndexPattern), + [layer, columnId, currentIndexPattern] ); const selectedColumn: IndexPatternColumn | null = layer.columns[props.columnId] ?? null; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index d4e58ee77b73b..534051ac02c16 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; -import { parseTimeShift, ShiftError } from '../../../../../../src/plugins/data/common'; +import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; @@ -118,7 +118,6 @@ export function TimeShift({ isFocused, activeData, layerId, - runtimeError, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -128,7 +127,6 @@ export function TimeShift({ isFocused: boolean; activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; - runtimeError?: Error | undefined; }) { const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { @@ -170,23 +168,9 @@ export function TimeShift({ ); } - function getRuntimeErrorReason() { - if (!(runtimeError instanceof ShiftError)) { - return undefined; - } - if (!runtimeError.aggIds.includes(columnId)) { - return undefined; - } - return runtimeError.reason; - } - const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = - (parsedLocalValue && isValueTooSmall(parsedLocalValue)) || - getRuntimeErrorReason() === 'tooSmall'; - const localValueNotMultiple = - (parsedLocalValue && isValueNotMultiple(parsedLocalValue)) || - getRuntimeErrorReason() === 'notAMultiple'; + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); const isLocalValueInvalid = Boolean( parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 179f8b2f1c5f2..43e386cff10a3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -49,8 +49,7 @@ export function isDraggedField(fieldCandidate: unknown): fieldCandidate is Dragg export function isColumnInvalid( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - runtimeError?: Error + indexPattern: IndexPattern ) { const column: IndexPatternColumn | undefined = layer.columns[columnId]; if (!column) return; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 5516f16e0bbae..4289366c2d399 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -299,7 +299,6 @@ export type DatasourceDimensionProps = SharedDimensionProps & { onRemove?: (accessor: string) => void; state: T; activeData?: Record; - runtimeError?: Error; }; // The only way a visualization has to restrict the query building @@ -321,7 +320,6 @@ export interface DatasourceLayerPanelProps { state: T; setState: StateSetter; activeData?: Record; - runtimeError?: Error; } export interface DraggedOperation extends DraggingIdentifier { @@ -522,7 +520,6 @@ export interface FramePublicAPI { * If accessing, make sure to check whether expected columns actually exist. */ activeData?: Record; - runtimeError?: Error; dateRange: DateRange; query: Query; From 0d2621e7b0e3764cd0ef9276a98320249d8c5a20 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 12:00:41 +0200 Subject: [PATCH 10/31] remove leftover runtime error stuff --- .../editor_frame/config_panel/layer_panel.tsx | 2 -- .../editor_frame_service/editor_frame/editor_frame.tsx | 2 -- .../editor_frame_service/editor_frame/state_management.ts | 8 -------- 3 files changed, 12 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 3b6e2b2a3eb02..cf3c9099d4b0d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -357,7 +357,6 @@ export function LayerPanel( columnId: accessorConfig.columnId, groupId: group.groupId, filterOperations: group.filterOperations, - runtimeError: props.framePublicAPI.runtimeError, }} /> @@ -434,7 +433,6 @@ export function LayerPanel( hideGrouping: activeGroup.hideGrouping, filterOperations: activeGroup.filterOperations, dimensionGroups: groups, - runtimeError: props.framePublicAPI.runtimeError, setState: ( newState: unknown, { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 642ef76da4f36..362787ea91c4f 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -110,7 +110,6 @@ export function EditorFrame(props: EditorFrameProps) { const framePublicAPI: FramePublicAPI = { datasourceLayers, activeData: state.activeData, - runtimeError: state.lastRuntimeError, dateRange: props.dateRange, query: props.query, filters: props.filters, @@ -367,7 +366,6 @@ export function EditorFrame(props: EditorFrameProps) { plugins={props.plugins} visualizeTriggerFieldContext={visualizeTriggerFieldContext} getSuggestionForField={getSuggestionForField} - hasRuntimeError={Boolean(state.lastRuntimeError)} /> ) } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index d49ee4cec46f3..53aba0d6f3f6c 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,7 +24,6 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; - lastRuntimeError?: Error; } export type Action = @@ -32,10 +31,6 @@ export type Action = type: 'RESET'; state: EditorFrameState; } - | { - type: 'RUNTIME_ERROR'; - error: Error; - } | { type: 'UPDATE_TITLE'; title: string; @@ -149,14 +144,11 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta return action.state; case 'UPDATE_TITLE': return { ...state, title: action.title }; - case 'RUNTIME_ERROR': - return { ...state, lastRuntimeError: action.error }; case 'UPDATE_STATE': return action.updater(state); case 'UPDATE_ACTIVE_DATA': return { ...state, - lastRuntimeError: undefined, activeData: { ...action.tables }, }; case 'UPDATE_LAYER': From 3439d351c28dbbc576724b95115a0f297904234e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 6 May 2021 18:08:50 +0200 Subject: [PATCH 11/31] fix some casest push --- .../data/common/search/aggs/agg_configs.ts | 118 ++++++------ .../buckets/_terms_other_bucket_helper.ts | 2 +- .../data/common/search/aggs/buckets/terms.ts | 8 +- .../operations/definitions/index.ts | 26 +-- .../operations/definitions/terms/index.tsx | 170 ++++++------------ .../operations/layer_helpers.ts | 10 +- 6 files changed, 124 insertions(+), 210 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 8dbde8b982586..180113b1b31d8 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -199,9 +199,12 @@ export class AggConfigs { let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; const timeShifts = this.getTimeShifts(); - const hasTimeShifts = Object.keys(timeShifts).length > 0; + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; if (this.hierarchical) { + if (hasMultipleTimeShifts) { + throw new Error('Multiple time shifts not supported for hierarchical metrics'); + } // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs .filter(function (agg) { @@ -228,7 +231,7 @@ export class AggConfigs { dslLvlCursor = prevDsl?.aggs || dslLvlCursor; } - if (hasTimeShifts) { + if (hasMultipleTimeShifts) { dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); } @@ -288,15 +291,6 @@ export class AggConfigs { const timeRange = this.timeRange; const filters: Record = {}; const timeField = this.timeFields[0]; - filters['0'] = { - range: { - [timeField]: { - // only works if there is a time range - gte: timeRange.from, - lte: timeRange.to, - }, - }, - }; Object.entries(timeShifts).forEach(([key, shift]) => { filters[key] = { range: { @@ -368,6 +362,8 @@ export class AggConfigs { .forEach((timeShift) => { if (timeShift) { timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } else { + timeShifts[0] = moment.duration(0); } }); return timeShifts; @@ -389,8 +385,7 @@ export class AggConfigs { const timeRange = this.timeRange; const timeFields = this.timeFields; const timeShifts = this.getTimeShifts(); - const hasTimeShift = Object.values(this.getTimeShifts()).length > 0; - if (!hasTimeShift) { + if (!this.hasTimeShifts()) { return this.timeFields .map((fieldName) => getTime(this.indexPattern, timeRange, { fieldName, forceNow })) .filter(isRangeFilter); @@ -406,43 +401,29 @@ export class AggConfigs { }, query: { bool: { - should: [ - ...Object.entries(timeShifts).map(([, shift]) => { - return { - bool: { - filter: timeFields - .map( - (fieldName) => - [ - getTime(this.indexPattern, timeRange, { fieldName, forceNow }), - fieldName, - ] as [RangeFilter | undefined, string] - ) - .filter(([filter]) => isRangeFilter(filter)) - .map(([filter, field]) => ({ - range: { - [field]: { - gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), - lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), - }, - }, - })), - }, - }; - }), - { + should: Object.entries(timeShifts).map(([, shift]) => { + return { bool: { filter: timeFields - .map((fieldName) => - getTime(this.indexPattern, timeRange, { fieldName, forceNow }) + .map( + (fieldName) => + [ + getTime(this.indexPattern, timeRange, { fieldName, forceNow }), + fieldName, + ] as [RangeFilter | undefined, string] ) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, + .filter(([filter]) => isRangeFilter(filter)) + .map(([filter, field]) => ({ + range: { + [field]: { + gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), + lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), + }, + }, })), }, - }, - ], + }; + }), minimum_should_match: 1, }, }, @@ -451,10 +432,11 @@ export class AggConfigs { } postFlightTransform(response: IEsSearchResponse) { - const timeShifts = this.getTimeShifts(); - if (Object.keys(timeShifts).length === 0) { + if (!this.hasTimeShifts()) { return response; } + const timeShifts = this.getTimeShifts(); + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); const aggCursor = transformedRawResponse.aggregations!; @@ -534,23 +516,33 @@ export class AggConfigs { }); }; const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - if (cursor.time_offset_split) { - const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< - string, - FiltersBucketItem - >; - const subTree = timeShiftedBuckets['0']; - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); + const shouldSplit = bucketAggs[aggIndex].splitForTimeShift(this); + if (shouldSplit) { + // multiple time shifts caused a filters agg in the tree we have to merge + if (hasMultipleTimeShifts && cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) + .buckets as Record; + const subTree = timeShiftedBuckets['0'] || {}; + Object.entries(timeShifts) + .filter(([key]) => key !== '0') + .forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); - delete cursor.time_offset_split; - Object.assign(cursor, subTree); + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + } else { + // otherwise we have to "merge" a single level to shift all keys + const [[, shift]] = Object.entries(timeShifts); + const subTree = {}; + mergeAggLevel(subTree, cursor, shift, aggIndex); + Object.assign(cursor, subTree); + } return; } // recurse deeper into the response object diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 6230ae897b170..372d487bcf7a3 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -166,7 +166,7 @@ export const buildOtherBucketAgg = ( key: string ) => { // make sure there are actually results for the buckets - if (aggregations[aggId].buckets.length < 1) { + if (aggregations[aggId]?.buckets.length < 1) { noAggBucketResults = true; return; } diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 44088ecb49db6..7356d43446358 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -180,12 +180,12 @@ export const getTermsBucketAgg = () => return; } - if (aggs?.hasTimeShifts()) { + if (aggs?.hasTimeShifts() && aggs.timeRange) { const shift = orderAgg.getTimeShift(); orderAgg = aggs.createAggConfig( { type: 'filtered_metric', - id: 'shifted-order', + id: orderAgg.id, params: { customBucket: aggs .createAggConfig( @@ -198,10 +198,10 @@ export const getTermsBucketAgg = () => query: { range: { [aggs.timeFields![0]]: { - from: moment(aggs.timeRange!.from) + gte: moment(aggs.timeRange.from) .subtract(shift || 0) .toISOString(), - to: moment(aggs.timeRange!.to) + lte: moment(aggs.timeRange.to) .subtract(shift || 0) .toISOString(), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 9421433eaf780..cefe4f1940e16 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -38,12 +38,7 @@ import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { FramePublicAPI, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; -import { - IndexPattern, - IndexPatternField, - IndexPatternLayer, - IndexPatternPrivateState, -} from '../../types'; +import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; @@ -227,10 +222,7 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - // TODO - remove these again and remap the state on a higher level - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) => | Array< | string @@ -238,10 +230,7 @@ interface BaseOperationDefinitionProps { message: string; fixAction?: { label: string; - newState: ( - core: CoreStart, - frame: FramePublicAPI - ) => Promise; + newState: (core: CoreStart, frame: FramePublicAPI) => Promise; }; } > @@ -345,9 +334,7 @@ interface FieldBasedOperationDefinition { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) => | Array< | string @@ -355,10 +342,7 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: ( - core: CoreStart, - frame: FramePublicAPI - ) => Promise; + newState: (core: CoreStart, frame: FramePublicAPI) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 9f42ba4daaf47..78b3a13c60e5e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -31,7 +31,7 @@ import { FiltersIndexPatternColumn, OperationDefinition, operationDefinitionMap import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; -import type { IndexPatternLayer, IndexPatternPrivateState } from '../../../types'; +import type { IndexPatternLayer, IndexPattern } from '../../../types'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -60,8 +60,7 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { function getDisallowedTermsMessage( layer: IndexPatternLayer, columnId: string, - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) { const hasMultipleShifts = uniq( @@ -77,123 +76,57 @@ function getDisallowedTermsMessage( defaultMessage: "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", }), - fixAction: - state && layerId - ? { - label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin current top values', + fixAction: { + label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { + defaultMessage: 'Pin current top values', + }), + newState: async (core: CoreStart, frame: FramePublicAPI) => { + const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; + const response: FieldStatsResponse = await core.http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, }), - newState: async (core: CoreStart, frame: FramePublicAPI) => { - const indexPattern = state.indexPatterns[layer.indexPatternId]; - const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await core.http.post( - `/api/lens/index_stats/${indexPattern.id}/field`, - { - body: JSON.stringify({ - fieldName, - dslQuery: esQuery.buildEsQuery( - indexPattern as IIndexPattern, - frame.query, - frame.filters, - esQuery.getEsQueryConfig(core.uiSettings) - ), - fromDate: frame.dateRange.fromDate, - toDate: frame.dateRange.toDate, - }), - } - ); - return { - ...state, - layers: { - ...state.layers, - [layerId]: { - ...layer, - columns: { - ...layer.columns, - [columnId]: { - label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { - defaultMessage: 'Pinned top values of {field}', - values: { - field: fieldName, - }, - }), - customLabel: true, - isBucketed: layer.columns[columnId].isBucketed, - dataType: 'string', - operationType: 'filters', - params: { - filters: response.topValues?.buckets.map(({ key }) => ({ - input: { - query: `${fieldName}: "${key}"`, - language: 'kuery', - }, - label: String(key), - })), - }, - } as FiltersIndexPatternColumn, - }, - }, - }, - }; - }, } - : undefined, - }; -} - -function getInvalidNestingOrderMessage( - layer: IndexPatternLayer, - columnId: string, - state?: IndexPatternPrivateState, - layerId?: string -) { - const usesTimeShift = Object.values(layer.columns).some( - (col) => col.timeShift && col.timeShift !== '' - ); - if (!usesTimeShift) { - return undefined; - } - const dateHistogramParent = layer.columnOrder - .slice(0, layer.columnOrder.indexOf(columnId)) - .find((colId) => layer.columns[colId].operationType === 'date_histogram'); - - if (!dateHistogramParent) { - return undefined; - } - return { - message: i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { - defaultMessage: - 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', - values: { - dateLabel: layer.columns[dateHistogramParent].label, - dimensionLabel: layer.columns[columnId].label, - }, - }), - fixAction: - state && layerId - ? { - label: i18n.translate( - 'xpack.lens.indexPattern.termsInShiftedDateHistogramFixActionLabel', - { - defaultMessage: 'Reorder dimensions', - } - ), - newState: async () => ({ - ...state, - layers: { - ...state.layers, - [layerId]: { - ...layer, - columns: { - ...layer.columns, - [dateHistogramParent]: layer.columns[columnId], - [columnId]: layer.columns[dateHistogramParent], - }, + ); + return { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { + defaultMessage: 'Pinned top values of {field}', + values: { + field: fieldName, }, + }), + customLabel: true, + isBucketed: layer.columns[columnId].isBucketed, + dataType: 'string', + operationType: 'filters', + params: { + filters: response.topValues?.buckets.map(({ key }) => ({ + input: { + query: `${fieldName}: "${key}"`, + language: 'kuery', + }, + label: String(key), + })), }, - }), - } - : undefined, + } as FiltersIndexPatternColumn, + }, + }; + }, + }, }; } @@ -236,14 +169,13 @@ export const termsOperation: OperationDefinition + getErrorMessage: (layer, columnId, indexPattern) => [ ...(getInvalidFieldMessage( layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern ) || []), - getInvalidNestingOrderMessage(layer, columnId, state, layerId) || '', - getDisallowedTermsMessage(layer, columnId, state, layerId) || '', + getDisallowedTermsMessage(layer, columnId, indexPattern) || '', ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index ffac563eeeea8..2942811670866 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1096,7 +1096,7 @@ export function getErrorMessages( .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); + return def.getErrorMessage(layer, columnId, indexPattern); } }) .map((errorMessage) => { @@ -1108,7 +1108,13 @@ export function getErrorMessages( fixAction: errorMessage.fixAction ? { ...errorMessage.fixAction, - newState: (frame: FramePublicAPI) => errorMessage.fixAction!.newState(core, frame), + newState: async (frame: FramePublicAPI) => ({ + ...state, + layers: { + ...state.layers, + [layerId]: await errorMessage.fixAction!.newState(core, frame), + }, + }), } : undefined, }; From abeed2c157bf1f56e5d0080b9db9f98008fabfb9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 11 May 2021 17:58:31 +0200 Subject: [PATCH 12/31] add tests and fix some bugs --- ...gins-data-public.aggconfig.gettimeshift.md | 15 + ...gins-data-public.aggconfig.hastimeshift.md | 15 + ...na-plugin-plugins-data-public.aggconfig.md | 2 + ...ic.aggconfigs.getsearchsourcetimefilter.md | 72 ++++ ...-public.aggconfigs.gettimeshiftinterval.md | 15 + ...ns-data-public.aggconfigs.gettimeshifts.md | 15 + ...ns-data-public.aggconfigs.hastimeshifts.md | 15 + ...a-plugin-plugins-data-public.aggconfigs.md | 5 + ...a-public.aggconfigs.postflighttransform.md | 22 ++ .../data/common/search/aggs/agg_config.ts | 22 +- .../common/search/aggs/agg_configs.test.ts | 338 ++++++++++++++++++ .../data/common/search/aggs/agg_configs.ts | 73 ++-- .../data/common/search/aggs/agg_type.ts | 19 +- .../search/aggs/buckets/bucket_agg_type.ts | 16 + .../common/search/aggs/metrics/avg_fn.test.ts | 1 + .../search/aggs/metrics/bucket_avg_fn.test.ts | 3 + .../search/aggs/metrics/bucket_avg_fn.ts | 7 + .../search/aggs/metrics/bucket_max_fn.test.ts | 3 + .../search/aggs/metrics/bucket_max_fn.ts | 7 + .../search/aggs/metrics/bucket_min_fn.test.ts | 3 + .../search/aggs/metrics/bucket_min_fn.ts | 7 + .../search/aggs/metrics/bucket_sum_fn.test.ts | 3 + .../search/aggs/metrics/bucket_sum_fn.ts | 7 + .../aggs/metrics/cardinality_fn.test.ts | 1 + .../search/aggs/metrics/count_fn.test.ts | 1 + .../aggs/metrics/cumulative_sum_fn.test.ts | 4 + .../search/aggs/metrics/cumulative_sum_fn.ts | 7 + .../search/aggs/metrics/derivative_fn.test.ts | 4 + .../search/aggs/metrics/derivative_fn.ts | 7 + .../aggs/metrics/filtered_metric_fn.test.ts | 3 + .../search/aggs/metrics/filtered_metric_fn.ts | 7 + .../search/aggs/metrics/geo_bounds_fn.test.ts | 1 + .../search/aggs/metrics/geo_bounds_fn.ts | 7 + .../aggs/metrics/geo_centroid_fn.test.ts | 1 + .../search/aggs/metrics/geo_centroid_fn.ts | 7 + .../common/search/aggs/metrics/max_fn.test.ts | 1 + .../data/common/search/aggs/metrics/median.ts | 2 +- .../search/aggs/metrics/median_fn.test.ts | 1 + .../search/aggs/metrics/metric_agg_type.ts | 4 +- .../common/search/aggs/metrics/min_fn.test.ts | 1 + .../aggs/metrics/percentile_ranks_fn.test.ts | 2 + .../aggs/metrics/percentile_ranks_fn.ts | 7 + .../aggs/metrics/percentiles_fn.test.ts | 2 + .../search/aggs/metrics/percentiles_fn.ts | 7 + .../aggs/metrics/serial_diff_fn.test.ts | 4 + .../search/aggs/metrics/serial_diff_fn.ts | 7 + .../aggs/metrics/std_deviation_fn.test.ts | 1 + .../search/aggs/metrics/std_deviation_fn.ts | 7 + .../common/search/aggs/metrics/sum_fn.test.ts | 1 + .../search/aggs/metrics/top_hit_fn.test.ts | 2 + src/plugins/data/public/public.api.md | 43 +++ src/plugins/data/server/server.api.md | 4 + .../snapshots/session/combined_test0.json | 1 - .../snapshots/session/combined_test1.json | 1 - .../snapshots/session/combined_test2.json | 1 - .../snapshots/session/combined_test3.json | 1 - .../snapshots/session/final_output_test.json | 1 - .../snapshots/session/metric_all_data.json | 1 - .../session/metric_invalid_data.json | 1 - .../session/metric_multi_metric_data.json | 1 - .../session/metric_percentage_mode.json | 1 - .../session/metric_single_metric_data.json | 1 - .../snapshots/session/partial_test_1.json | 1 - .../snapshots/session/partial_test_2.json | 1 - .../snapshots/session/partial_test_3.json | 1 - .../snapshots/session/step_output_test0.json | 1 - .../snapshots/session/step_output_test1.json | 1 - .../snapshots/session/step_output_test2.json | 1 - .../snapshots/session/step_output_test3.json | 1 - .../snapshots/session/tagcloud_all_data.json | 1 - .../snapshots/session/tagcloud_fontsize.json | 1 - .../session/tagcloud_invalid_data.json | 1 - .../session/tagcloud_metric_data.json | 1 - .../snapshots/session/tagcloud_options.json | 1 - .../run_pipeline/esaggs_timeshift.ts | 198 ++++++++++ .../test_suites/run_pipeline/index.ts | 1 + .../editor_frame/frame_layout.tsx | 2 +- .../workspace_panel/workspace_panel.tsx | 1 + .../dimension_panel/dimension_editor.tsx | 6 +- .../dimension_panel/dimension_panel.test.tsx | 194 ++++++++++ .../dimension_panel/time_shift.tsx | 22 +- .../indexpattern.test.ts | 37 ++ .../operations/definitions/date_histogram.tsx | 58 +-- .../definitions/terms/terms.test.tsx | 105 +++++- .../indexpattern_datasource/to_expression.ts | 2 +- x-pack/test/functional/apps/lens/index.ts | 1 + .../test/functional/apps/lens/time_shift.ts | 68 ++++ .../test/functional/page_objects/lens_page.ts | 19 + 88 files changed, 1404 insertions(+), 163 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md delete mode 100644 test/interpreter_functional/snapshots/session/combined_test0.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test1.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test2.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test3.json delete mode 100644 test/interpreter_functional/snapshots/session/final_output_test.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_all_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_invalid_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_multi_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_percentage_mode.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_single_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_1.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_2.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_3.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test0.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test1.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test2.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test3.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_all_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_fontsize.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_options.json create mode 100644 test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts create mode 100644 x-pack/test/functional/apps/lens/time_shift.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md new file mode 100644 index 0000000000000..de0d41286c0bb --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) > [getTimeShift](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md) + +## AggConfig.getTimeShift() method + +Signature: + +```typescript +getTimeShift(): undefined | moment.Duration; +``` +Returns: + +`undefined | moment.Duration` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md new file mode 100644 index 0000000000000..024b0766ffd7b --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) > [hasTimeShift](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md) + +## AggConfig.hasTimeShift() method + +Signature: + +```typescript +hasTimeShift(): boolean; +``` +Returns: + +`boolean` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md index d4a8eddf51cfc..a96626d1a485d 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md @@ -46,8 +46,10 @@ export declare class AggConfig | [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfig.getrequestaggs.md) | | | | [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfig.getresponseaggs.md) | | | | [getTimeRange()](./kibana-plugin-plugins-data-public.aggconfig.gettimerange.md) | | | +| [getTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md) | | | | [getValue(bucket)](./kibana-plugin-plugins-data-public.aggconfig.getvalue.md) | | | | [getValueBucketPath()](./kibana-plugin-plugins-data-public.aggconfig.getvaluebucketpath.md) | | Returns the bucket path containing the main value the agg will produce (e.g. for sum of bytes it will point to the sum, for median it will point to the 50 percentile in the percentile multi value bucket) | +| [hasTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md) | | | | [isFilterable()](./kibana-plugin-plugins-data-public.aggconfig.isfilterable.md) | | | | [makeLabel(percentageMode)](./kibana-plugin-plugins-data-public.aggconfig.makelabel.md) | | | | [nextId(list)](./kibana-plugin-plugins-data-public.aggconfig.nextid.md) | static | Calculate the next id based on the ids in this list {array} list - a list of objects with id properties | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md new file mode 100644 index 0000000000000..1f8bc1300a0a8 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md @@ -0,0 +1,72 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getSearchSourceTimeFilter](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md) + +## AggConfigs.getSearchSourceTimeFilter() method + +Signature: + +```typescript +getSearchSourceTimeFilter(forceNow?: Date): RangeFilter[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| forceNow | Date | | + +Returns: + +`RangeFilter[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md new file mode 100644 index 0000000000000..d15ccbc5dc0a1 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getTimeShiftInterval](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md) + +## AggConfigs.getTimeShiftInterval() method + +Signature: + +```typescript +getTimeShiftInterval(): moment.Duration | undefined; +``` +Returns: + +`moment.Duration | undefined` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md new file mode 100644 index 0000000000000..44ab25cf30eb2 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md) + +## AggConfigs.getTimeShifts() method + +Signature: + +```typescript +getTimeShifts(): Record; +``` +Returns: + +`Record` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md new file mode 100644 index 0000000000000..db31e549666b4 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [hasTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md) + +## AggConfigs.hasTimeShifts() method + +Signature: + +```typescript +hasTimeShifts(): boolean; +``` +Returns: + +`boolean` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index 02e9a63d95ba3..ca191fdcae9a5 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -43,8 +43,13 @@ export declare class AggConfigs | [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getrequestaggs.md) | | | | [getResponseAggById(id)](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggbyid.md) | | Find a response agg by it's id. This may be an agg in the aggConfigs, or one created specifically for a response value | | [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggs.md) | | Gets the AggConfigs (and possibly ResponseAggConfigs) that represent the values that will be produced when all aggs are run.With multi-value metric aggs it is possible for a single agg request to result in multiple agg values, which is why the length of a vis' responseValuesAggs may be different than the vis' aggs {array\[AggConfig\]} | +| [getSearchSourceTimeFilter(forceNow)](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md) | | | +| [getTimeShiftInterval()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md) | | | +| [getTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md) | | | +| [hasTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md) | | | | [jsonDataEquals(aggConfigs)](./kibana-plugin-plugins-data-public.aggconfigs.jsondataequals.md) | | Data-by-data comparison of this Aggregation Ignores the non-array indexes | | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | +| [postFlightTransform(response)](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | | [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md new file mode 100644 index 0000000000000..b34fda40a3089 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [postFlightTransform](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) + +## AggConfigs.postFlightTransform() method + +Signature: + +```typescript +postFlightTransform(response: IEsSearchResponse): IEsSearchResponse; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| response | IEsSearchResponse<any> | | + +Returns: + +`IEsSearchResponse` + diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 5c34e17fb5c0b..3c83b5bdf6084 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -20,7 +20,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; -import { GenericBucket, IAggConfigs } from './agg_configs'; +import { IAggConfigs } from './agg_configs'; import { parseTimeShift } from './utils'; type State = string | number | boolean | null | undefined | SerializableState; @@ -199,26 +199,6 @@ export class AggConfig { return parsedTimeShift; } - getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { - return this.type.getShiftedKey(this, key, timeShift); - } - - getTimeShiftInterval(): undefined | moment.Duration { - return this.type.getTimeShiftInterval(this); - } - - splitForTimeShift(aggs: IAggConfigs) { - return this.type.splitForTimeShift(this, aggs); - } - - orderBuckets(a: GenericBucket, b: GenericBucket): number { - if (this.type.orderBuckets) { - return this.type.orderBuckets(this, a, b); - } else { - return 0; - } - } - write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index 28102544ae055..d4980447c5468 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -14,6 +14,7 @@ import { mockAggTypesRegistry } from './test_helpers'; import type { IndexPatternField } from '../../index_patterns'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { stubIndexPattern, stubIndexPatternWithFields } from '../../../common/stubs'; +import { IEsSearchResponse } from '..'; describe('AggConfigs', () => { let indexPattern: IndexPattern; @@ -332,6 +333,101 @@ describe('AggConfigs', () => { }); }); + it('inserts a time split filters agg if there are multiple time shifts', () => { + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes', timeShift: '1d' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const dsl = ac.toDsl(); + + const terms = ac.byName('terms')[0]; + const avg = ac.byName('avg')[0]; + const sum = ac.byName('sum')[0]; + + expect(dsl[terms.id].aggs.time_offset_split.filters.filters).toMatchInlineSnapshot(` + Object { + "0": Object { + "range": Object { + "timestamp": Object { + "gte": "2021-05-05T00:00:00.000Z", + "lte": "2021-05-10T00:00:00.000Z", + }, + }, + }, + "86400000": Object { + "range": Object { + "timestamp": Object { + "gte": "2021-05-04T00:00:00.000Z", + "lte": "2021-05-09T00:00:00.000Z", + }, + }, + }, + } + `); + expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(avg.id); + expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(sum.id); + }); + + it('does not insert a time split if there is a single time shift', () => { + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes', timeShift: '1d' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const dsl = ac.toDsl(); + + const terms = ac.byName('terms')[0]; + const avg = ac.byName('avg')[0]; + const sum = ac.byName('sum')[0]; + + expect(dsl[terms.id].aggs).not.toHaveProperty('time_offset_split'); + expect(dsl[terms.id].aggs).toHaveProperty(avg.id); + expect(dsl[terms.id].aggs).toHaveProperty(sum.id); + }); + it('writes multiple metric aggregations at every level if the vis is hierarchical', () => { const configStates = [ { enabled: true, type: 'terms', schema: 'segment', params: { field: 'bytes', orderBy: 1 } }, @@ -426,4 +522,246 @@ describe('AggConfigs', () => { ); }); }); + + describe('#postFlightTransform', () => { + it('merges together splitted responses for multiple shifts', () => { + indexPattern = stubIndexPattern as IndexPattern; + indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField); + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { + enabled: true, + type: 'date_histogram', + schema: 'segment', + params: { field: '@timestamp', interval: '1d' }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['@timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + // 1 terms bucket (A), with 2 date buckets (7th and 8th of May) + // the bucket keys of the shifted time range will be shifted forward + const response = { + rawResponse: { + aggregations: { + '1': { + buckets: [ + { + key: 'A', + time_offset_split: { + buckets: { + '0': { + 2: { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + 3: { + value: 1.1, + }, + 4: { + value: 2.2, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + doc_count: 26, + 3: { + value: 3.3, + }, + 4: { + value: 4.4, + }, + }, + ], + }, + }, + '86400000': { + 2: { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + doc_count: 13, + 3: { + value: 5.5, + }, + 4: { + value: 6.6, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + 3: { + value: 7.7, + }, + 4: { + value: 8.8, + }, + }, + ], + }, + }, + }, + }, + }, + ], + }, + }, + }, + }; + const mergedResponse = ac.postFlightTransform( + (response as unknown) as IEsSearchResponse + ); + expect(mergedResponse.rawResponse).toEqual({ + aggregations: { + '1': { + buckets: [ + { + '2': { + buckets: [ + { + '4': { + value: 2.2, + }, + // 2021-05-07 + key: 1620345600000, + }, + { + '3': { + value: 5.5, + }, + '4': { + value: 4.4, + }, + doc_count: 26, + doc_count_86400000: 13, + // 2021-05-08 + key: 1620432000000, + }, + { + '3': { + value: 7.7, + }, + // 2021-05-09 + key: 1620518400000, + }, + ], + }, + key: 'A', + }, + ], + }, + }, + }); + }); + + it('shifts date histogram keys and renames doc_count properties for single shift', () => { + indexPattern = stubIndexPattern as IndexPattern; + indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField); + const configStates = [ + { + enabled: true, + type: 'date_histogram', + schema: 'segment', + params: { field: '@timestamp', interval: '1d' }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['@timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const response = { + rawResponse: { + aggregations: { + '1': { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + doc_count: 26, + 2: { + value: 1.1, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + doc_count: 27, + 2: { + value: 2.2, + }, + }, + ], + }, + }, + }, + }; + const mergedResponse = ac.postFlightTransform( + (response as unknown) as IEsSearchResponse + ); + expect(mergedResponse.rawResponse).toEqual({ + aggregations: { + '1': { + buckets: [ + { + '2': { + value: 1.1, + }, + doc_count_86400000: 26, + // 2021-05-08 + key: 1620432000000, + }, + { + '2': { + value: 2.2, + }, + doc_count_86400000: 27, + // 2021-05-09 + key: 1620518400000, + }, + ], + }, + }, + }); + }); + }); }); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 180113b1b31d8..bab7fab5f2248 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,6 +30,7 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; +import { IBucketAggConfig } from './buckets'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -217,7 +218,13 @@ export class AggConfigs { }; }); } - this.getRequestAggs().forEach((config: AggConfig, i: number, list) => { + const requestAggs = this.getRequestAggs(); + const aggsWithDsl = requestAggs.filter((agg) => !agg.type.hasNoDsl).length; + const timeSplitIndex = this.getAll().findIndex( + (config) => 'splitForTimeShift' in config.type && config.type.splitForTimeShift(config, this) + ); + + requestAggs.forEach((config: AggConfig, i: number, list) => { if (!dslLvlCursor) { // start at the top level dslLvlCursor = dslTopLvl; @@ -246,8 +253,11 @@ export class AggConfigs { parseParentAggs(dslLvlCursor, dsl); - if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { - // buckets that are not the last item in the list accept sub-aggs + if ( + config.type.type === AggGroupNames.Buckets && + (i < aggsWithDsl - 1 || timeSplitIndex > i) + ) { + // buckets that are not the last item in the list of dsl producing aggs or have a time split coming up accept sub-aggs subAggs = dsl.aggs || (dsl.aggs = {}); } @@ -279,7 +289,7 @@ export class AggConfigs { timeShifts: Record, dslLvlCursor: Record ) { - if (!config.splitForTimeShift(this)) { + if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, this)) { return dslLvlCursor; } if (!this.timeFields || this.timeFields.length < 1) { @@ -370,8 +380,10 @@ export class AggConfigs { } getTimeShiftInterval(): moment.Duration | undefined { - const splitAgg = this.getAll().find((agg) => agg.splitForTimeShift(this)); - return splitAgg?.getTimeShiftInterval(); + const splitAgg = (this.getAll().filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]).find((agg) => agg.type.splitForTimeShift(agg, this)); + return splitAgg?.type.getTimeShiftInterval(splitAgg); } hasTimeShifts(): boolean { @@ -438,9 +450,16 @@ export class AggConfigs { const timeShifts = this.getTimeShifts(); const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); + if (!transformedRawResponse.aggregations) { + transformedRawResponse.aggregations = { + doc_count: response.rawResponse.hits?.total as Aggregate, + }; + } const aggCursor = transformedRawResponse.aggregations!; - const bucketAggs = this.aggs.filter((agg) => agg.type.type === AggGroupNames.Buckets); + const bucketAggs = this.aggs.filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]; const mergeAggLevel = ( target: GenericBucket, @@ -451,7 +470,11 @@ export class AggConfigs { Object.entries(source).forEach(([key, val]) => { // copy over doc count into special key if (typeof val === 'number' && key === 'doc_count') { - target[`doc_count_${shift.asMilliseconds()}`] = val; + if (shift.asMilliseconds() === 0) { + target.doc_count = val; + } else { + target[`doc_count_${shift.asMilliseconds()}`] = val; + } } else if (typeof val !== 'object') { // other meta keys not of interest return; @@ -460,11 +483,15 @@ export class AggConfigs { const agg = this.byId(key); if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); - if (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) { + if ( + (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || + (shift.asMilliseconds() === 0 && !timeShift) + ) { // this is a metric from the current time shift, copy it over target[key] = source[key]; } - } else if (agg === bucketAggs[aggIndex]) { + } else if (agg && agg === bucketAggs[aggIndex]) { + const bucketAgg = agg as IBucketAggConfig; // expected next bucket sub agg const subAggregate = val as Aggregate; const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as @@ -490,7 +517,7 @@ export class AggConfigs { baseBucketMap[String(bucket.key)] = bucket; }); buckets.forEach((bucket) => { - const bucketKey = agg.getShiftedKey(bucket.key, shift); + const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); // if a bucket is missing in the map, create an empty one if (!baseBucketMap[bucketKey]) { baseBucketMap[String(bucketKey)] = { @@ -501,7 +528,7 @@ export class AggConfigs { }); (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( baseBucketMap - ).sort(agg.orderBuckets.bind(agg)); + ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); } else if (baseBuckets && buckets && !isArray(baseBuckets)) { Object.entries(buckets).forEach(([bucketKey, bucket]) => { // if a bucket is missing in the base response, create an empty one @@ -516,23 +543,21 @@ export class AggConfigs { }); }; const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - const shouldSplit = bucketAggs[aggIndex].splitForTimeShift(this); + const shouldSplit = this.aggs[aggIndex].type.splitForTimeShift(this.aggs[aggIndex], this); if (shouldSplit) { // multiple time shifts caused a filters agg in the tree we have to merge if (hasMultipleTimeShifts && cursor.time_offset_split) { const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) .buckets as Record; - const subTree = timeShiftedBuckets['0'] || {}; - Object.entries(timeShifts) - .filter(([key]) => key !== '0') - .forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); + const subTree = {}; + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); delete cursor.time_offset_split; Object.assign(cursor, subTree); diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index 4d30d6a211650..48ce54bbd61bd 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -8,7 +8,6 @@ import { constant, noop, identity } from 'lodash'; import { i18n } from '@kbn/i18n'; -import moment from 'moment'; import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; @@ -17,7 +16,7 @@ import type { RequestAdapter } from 'src/plugins/inspector/common'; import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; -import { GenericBucket, IAggConfigs } from './agg_configs'; +import { IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; @@ -216,26 +215,10 @@ export class AggType< return agg.id; }; - getShiftedKey( - agg: TAggConfig, - key: string | number, - timeShift: moment.Duration - ): string | number { - return key; - } - splitForTimeShift(agg: TAggConfig, aggs: IAggConfigs) { return false; } - getTimeShiftInterval(agg: TAggConfig): undefined | moment.Duration { - return undefined; - } - - orderBuckets(agg: TAggConfig, a: GenericBucket, b: GenericBucket): number { - return Number(a.key) - Number(b.key); - } - /** * Generic AggType Constructor * diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts index 1626a89e9a331..d44e634a00fe6 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts @@ -44,6 +44,22 @@ export class BucketAggType any; type = bucketType; + getShiftedKey( + agg: TBucketAggConfig, + key: string | number, + timeShift: moment.Duration + ): string | number { + return key; + } + + getTimeShiftInterval(agg: TBucketAggConfig): undefined | moment.Duration { + return undefined; + } + + orderBuckets(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number { + return Number(a.key) - Number(b.key); + } + constructor(config: BucketAggTypeConfig) { super(config); diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts index 05a6e9eeff7d7..0b794617fb96e 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg", diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts index 33f20b9a40dc2..ac214c1a1591c 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts index b79e57207ebd8..b9374fb94720a 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts @@ -77,6 +77,13 @@ export const aggBucketAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts index 35b765ec0e075..e6db7665a68dd 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts index e12a592448334..f50042bc1ca10 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts @@ -77,6 +77,13 @@ export const aggBucketMax = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts index 49346036ce649..22ec55506fe90 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts index ece5c07c6e5f8..41bec5c5746c3 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts @@ -77,6 +77,13 @@ export const aggBucketMin = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts index 0f5c84a477b06..0e3370cec14e5 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts index 5fe0ee75bfe38..b56dba3b96ba1 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts @@ -77,6 +77,13 @@ export const aggBucketSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts index 8b235edacb59a..08d64e599d8a9 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cardinality", diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts index 047b9bbd8517f..c6736c5b69f7d 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts @@ -23,6 +23,7 @@ describe('agg_expression_functions', () => { "id": undefined, "params": Object { "customLabel": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "count", diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts index 5eb6d2b780442..f311ab35a8d0d 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts index cba4de1ad11ae..88c1fceb11f72 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts @@ -81,6 +81,13 @@ export const aggCumulativeSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts index 1eaca811a2481..3e4fc838dd398 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "derivative", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "derivative", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "derivative", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts index e27179c7209ad..38985b78cb378 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts @@ -81,6 +81,13 @@ export const aggDerivative = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts index 22e97fe18b604..d1ce6ff463903 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts @@ -28,6 +28,7 @@ describe('agg_expression_functions', () => { "customBucket": undefined, "customLabel": undefined, "customMetric": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "filtered_metric", @@ -40,10 +41,12 @@ describe('agg_expression_functions', () => { "customBucket": undefined, "customLabel": undefined, "customMetric": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "filtered_metric", }, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts index 6a7ff5fa5fd40..7b14fd1c45ed2 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts @@ -72,6 +72,13 @@ export const aggFilteredMetric = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts index c48233e84404c..50b5f5b60376b 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "geo_bounds", diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts index 19d2dabc843dd..c7915eb12c5b2 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts @@ -67,6 +67,13 @@ export const aggGeoBounds = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts index e984df13527ca..889ed29c63ee1 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "geo_centroid", diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts index 1cc11c345e9ba..df1b5182092fd 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts @@ -67,6 +67,13 @@ export const aggGeoCentroid = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts index d94e01927c851..021c5aac69e10 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max", diff --git a/src/plugins/data/common/search/aggs/metrics/median.ts b/src/plugins/data/common/search/aggs/metrics/median.ts index bad4c7baf173f..4fdb1ce6b7d81 100644 --- a/src/plugins/data/common/search/aggs/metrics/median.ts +++ b/src/plugins/data/common/search/aggs/metrics/median.ts @@ -46,7 +46,7 @@ export const getMedianMetricAgg = () => { { name: 'percents', default: [50], shouldShow: () => false, serialize: () => undefined }, ], getValue(agg, bucket) { - return bucket[agg.id].values['50.0']; + return bucket[agg.id]?.values['50.0']; }, }); }; diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts index e70520b743e17..7ff7f18cdbc02 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "median", diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 9186ba8f70cfc..6ddb0fdd9410d 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -83,7 +83,9 @@ export class MetricAggType aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg && - !aggs.byType(AggGroupNames.Buckets).some((bucketAgg) => bucketAgg.splitForTimeShift(aggs)); + !aggs + .byType(AggGroupNames.Buckets) + .some((bucketAgg) => bucketAgg.type.splitForTimeShift(bucketAgg, aggs)); } } diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts index ea2d2cd23edae..fee4b28882408 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min", diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts index 9328597b24cfa..873765374c80a 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, "values": undefined, }, "schema": undefined, @@ -51,6 +52,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, "values": Array [ 1, 2, diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts index 7929a01c0b589..7bbf8c460682b 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts @@ -74,6 +74,13 @@ export const aggPercentileRanks = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts index 0d71df240d122..468da036cea88 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts @@ -28,6 +28,7 @@ describe('agg_expression_functions', () => { "field": "machine.os.keyword", "json": undefined, "percents": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "percentiles", @@ -56,6 +57,7 @@ describe('agg_expression_functions', () => { 2, 3, ], + "timeShift": undefined, }, "schema": undefined, "type": "percentiles", diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts index fa5120dfc3b97..8c9c765d068e2 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts @@ -74,6 +74,13 @@ export const aggPercentiles = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts index 065ef8021cbda..aa73d5c44dd7f 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts index 925d85774c7ad..b481fa56a38ce 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts @@ -81,6 +81,13 @@ export const aggSerialDiff = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts index 9aaf82e65812b..849987695dc7c 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "std_dev", diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts index 80787a3383c6b..7ba0c5b789dbf 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts @@ -67,6 +67,13 @@ export const aggStdDeviation = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts index e19fc072e1cd9..f4d4fb5451dcd 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum", diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts index e9d6a619a9cd6..2f8ef74b5c2f0 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts @@ -32,6 +32,7 @@ describe('agg_expression_functions', () => { "size": undefined, "sortField": undefined, "sortOrder": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "top_hits", @@ -64,6 +65,7 @@ describe('agg_expression_functions', () => { "size": 6, "sortField": "_score", "sortOrder": "asc", + "timeShift": undefined, }, "schema": "whatever", "type": "top_hits", diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index d4f0ccfe810c6..16ed4d19e2ed9 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -7,11 +7,13 @@ import { $Values } from '@kbn/utility-types'; import { Action } from 'history'; import { Adapters as Adapters_2 } from 'src/plugins/inspector/common'; +import { Aggregate } from '@elastic/elasticsearch/api/types'; import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; import { ApplicationStart } from 'kibana/public'; import { Assign } from '@kbn/utility-types'; import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import Boom from '@hapi/boom'; +import { Bucket } from '@elastic/elasticsearch/api/types'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/public'; import { CoreSetup as CoreSetup_2 } from 'kibana/public'; @@ -46,6 +48,7 @@ import { Href } from 'history'; import { HttpSetup } from 'kibana/public'; import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { IconType } from '@elastic/eui'; +import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public'; import { IncomingHttpHeaders } from 'http'; import { InjectedIntl } from '@kbn/i18n/react'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; @@ -74,6 +77,7 @@ import * as PropTypes from 'prop-types'; import { PublicContract } from '@kbn/utility-types'; import { PublicMethodsOf } from '@kbn/utility-types'; import { PublicUiSettingsParams } from 'src/core/server/types'; +import { RangeFilter as RangeFilter_2 } from 'src/plugins/data/public'; import React from 'react'; import * as React_3 from 'react'; import { RecursiveReadonly } from '@kbn/utility-types'; @@ -152,9 +156,13 @@ export class AggConfig { // (undocumented) getTimeRange(): import("../../../public").TimeRange | undefined; // (undocumented) + getTimeShift(): undefined | moment.Duration; + // (undocumented) getValue(bucket: any): any; getValueBucketPath(): string; // (undocumented) + hasTimeShift(): boolean; + // (undocumented) id: string; // (undocumented) isFilterable(): boolean; @@ -253,6 +261,39 @@ export class AggConfigs { getResponseAggById(id: string): AggConfig | undefined; getResponseAggs(): AggConfig[]; // (undocumented) + getSearchSourceTimeFilter(forceNow?: Date): RangeFilter_2[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]; + // (undocumented) + getTimeShiftInterval(): moment.Duration | undefined; + // (undocumented) + getTimeShifts(): Record; + // (undocumented) + hasTimeShifts(): boolean; + // (undocumented) hierarchical?: boolean; // (undocumented) indexPattern: IndexPattern; @@ -260,6 +301,8 @@ export class AggConfigs { // (undocumented) onSearchRequestStart(searchSource: ISearchSource_2, options?: ISearchOptions_2): Promise<[unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown]>; // (undocumented) + postFlightTransform(response: IEsSearchResponse_2): IEsSearchResponse_2; + // (undocumented) setTimeFields(timeFields: string[] | undefined): void; // (undocumented) setTimeRange(timeRange: TimeRange): void; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index e66eaab672e1c..f9f155db5751e 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -6,8 +6,10 @@ import { $Values } from '@kbn/utility-types'; import { Adapters } from 'src/plugins/inspector/common'; +import { Aggregate } from '@elastic/elasticsearch/api/types'; import { Assign } from '@kbn/utility-types'; import { BfetchServerSetup } from 'src/plugins/bfetch/server'; +import { Bucket } from '@elastic/elasticsearch/api/types'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/server'; import { CoreSetup as CoreSetup_2 } from 'kibana/server'; @@ -32,6 +34,7 @@ import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ExpressionValueBoxed } from 'src/plugins/expressions/common'; import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_formats/utils'; import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; +import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public'; import { IScopedClusterClient } from 'src/core/server'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; import { ISearchSource } from 'src/plugins/data/public'; @@ -52,6 +55,7 @@ import { Plugin as Plugin_2 } from 'src/core/server'; import { Plugin as Plugin_3 } from 'kibana/server'; import { PluginInitializerContext as PluginInitializerContext_2 } from 'src/core/server'; import { PublicMethodsOf } from '@kbn/utility-types'; +import { RangeFilter } from 'src/plugins/data/public'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; diff --git a/test/interpreter_functional/snapshots/session/combined_test0.json b/test/interpreter_functional/snapshots/session/combined_test0.json deleted file mode 100644 index 8f00d72df8ab3..0000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test0.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test1.json b/test/interpreter_functional/snapshots/session/combined_test1.json deleted file mode 100644 index 8f00d72df8ab3..0000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test1.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test2.json b/test/interpreter_functional/snapshots/session/combined_test2.json deleted file mode 100644 index 4870694e6adbc..0000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test2.json +++ /dev/null @@ -1 +0,0 @@ -{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test3.json b/test/interpreter_functional/snapshots/session/combined_test3.json deleted file mode 100644 index 64b1052552c8f..0000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/final_output_test.json b/test/interpreter_functional/snapshots/session/final_output_test.json deleted file mode 100644 index 64b1052552c8f..0000000000000 --- a/test/interpreter_functional/snapshots/session/final_output_test.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_all_data.json b/test/interpreter_functional/snapshots/session/metric_all_data.json deleted file mode 100644 index 0e1e5a723373f..0000000000000 --- a/test/interpreter_functional/snapshots/session/metric_all_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":2,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_invalid_data.json b/test/interpreter_functional/snapshots/session/metric_invalid_data.json deleted file mode 100644 index c7b4a0325dc91..0000000000000 --- a/test/interpreter_functional/snapshots/session/metric_invalid_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json deleted file mode 100644 index fc8622a818dec..0000000000000 --- a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json deleted file mode 100644 index 95c011f9259b9..0000000000000 --- a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":1000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":true,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json deleted file mode 100644 index f4a8cd1f14e18..0000000000000 --- a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_1.json b/test/interpreter_functional/snapshots/session/partial_test_1.json deleted file mode 100644 index 14c8428c6d432..0000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_1.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_2.json b/test/interpreter_functional/snapshots/session/partial_test_2.json deleted file mode 100644 index 64b1052552c8f..0000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_2.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_3.json b/test/interpreter_functional/snapshots/session/partial_test_3.json deleted file mode 100644 index e011b69de2022..0000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"region_map_vis","type":"render","value":{"visConfig":{"addTooltip":true,"bucket":{"accessor":0},"colorSchema":"Yellow to Red","isDisplayWarning":true,"legendPosition":"bottomright","mapCenter":[0,0],"mapZoom":2,"metric":{"accessor":1,"format":{"id":"number"}},"outlineWeight":1,"selectedJoinField":{},"selectedLayer":{},"showAllShapes":true,"wms":{}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"region_map"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test0.json b/test/interpreter_functional/snapshots/session/step_output_test0.json deleted file mode 100644 index 8f00d72df8ab3..0000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test0.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test1.json b/test/interpreter_functional/snapshots/session/step_output_test1.json deleted file mode 100644 index 8f00d72df8ab3..0000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test1.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test2.json b/test/interpreter_functional/snapshots/session/step_output_test2.json deleted file mode 100644 index 4870694e6adbc..0000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test2.json +++ /dev/null @@ -1 +0,0 @@ -{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test3.json b/test/interpreter_functional/snapshots/session/step_output_test3.json deleted file mode 100644 index 64b1052552c8f..0000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json deleted file mode 100644 index 073fca760b9a2..0000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json deleted file mode 100644 index 93f8d8a27d233..0000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json deleted file mode 100644 index 0c50947beca97..0000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json deleted file mode 100644 index e8c47efdbe622..0000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_options.json b/test/interpreter_functional/snapshots/session/tagcloud_options.json deleted file mode 100644 index 38683082975f8..0000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_options.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts new file mode 100644 index 0000000000000..7971a7b8ee13a --- /dev/null +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { Datatable } from 'src/plugins/expressions'; +import { ExpectExpression, expectExpressionProvider } from './helpers'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +function getCell(esaggsResult: any, row: number, column: number): unknown | undefined { + const columnId = esaggsResult?.columns[column]?.id; + if (!columnId) { + return; + } + return esaggsResult?.rows[row]?.[columnId]; +} + +function checkShift(rows: Datatable['rows'], columns: Datatable['columns'], metricIndex = 1) { + rows.shift(); + rows.pop(); + rows.forEach((_, index) => { + if (index < rows.length - 1) { + expect(getCell({ rows, columns }, index, metricIndex + 1)).to.be( + getCell({ rows, columns }, index + 1, metricIndex) + ); + } + }); +} + +export default function ({ + getService, + updateBaselines, +}: FtrProviderContext & { updateBaselines: boolean }) { + let expectExpression: ExpectExpression; + + describe('esaggs timeshift tests', () => { + before(() => { + expectExpression = expectExpressionProvider({ getService, updateBaselines }); + }); + + const timeRange = { + from: '2015-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + + it('shifts single metric', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4763); + }); + + it('shifts multiple metrics', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggCount id="1" enabled=true schema="metric" timeShift="12h"} + aggs={aggCount id="2" enabled=true schema="metric" timeShift="1d"} + aggs={aggCount id="3" enabled=true schema="metric"} + `; + const result = await expectExpression('esaggs_shift_multi_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4629); + expect(getCell(result, 0, 1)).to.be(4763); + expect(getCell(result, 0, 2)).to.be(4618); + }); + + it('shifts date histogram', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"} + aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1h"} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_date_histogram', + expression + ).getResponse(); + expect(result.rows.length).to.be(25); + checkShift(result.rows, result.columns); + }); + + it('shifts terms', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggTerms id="1" field="geo.src" size="3" enabled=true schema="bucket" orderAgg={aggCount id="order" enabled=true schema="metric"} otherBucket=true} + aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1d"} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_terms', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 'CN', + 'col-1-2': 40, + 'col-2-3': 5806.404352806415, + }, + { + 'col-0-1': 'IN', + 'col-1-2': 7901, + 'col-2-3': 5838.315923566879, + }, + { + 'col-0-1': 'US', + 'col-1-2': 7440, + 'col-2-3': 5614.142857142857, + }, + { + 'col-0-1': '__other__', + 'col-1-2': 5766.575645756458, + 'col-2-3': 5742.1265576323985, + }, + ]); + }); + + it('shifts histogram', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggHistogram id="1" field="bytes" interval=5000 enabled=true schema="bucket"} + aggs={aggCount id="2" enabled=true schema="metric"} + aggs={aggCount id="3" enabled=true schema="metric" timeShift="6h"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_histogram', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 0, + 'col-1-2': 2020, + 'col-2-3': 2036, + }, + { + 'col-0-1': 5000, + 'col-1-2': 2360, + 'col-2-3': 2358, + }, + { + 'col-0-1': 10000, + 'col-1-2': 126, + 'col-2-3': 127, + }, + { + 'col-0-1': 15000, + 'col-1-2': 112, + 'col-2-3': 108, + }, + ]); + }); + + it('shifts pipeline aggs', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggBucketSum id="1" enabled=true schema="metric" customBucket={aggTerms id="2" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="4" enabled="true" schema="metric"}} + aggs={aggBucketSum id="5" enabled=true schema="metric" timeShift="1d" customBucket={aggTerms id="6" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="7" enabled="true" schema="metric"}} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_pipeline_aggs', + expression + ).getResponse(); + expect(getCell(result, 0, 0)).to.be(2050); + expect(getCell(result, 0, 1)).to.be(2053); + }); + + it('metrics at all levels should work for single shift', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4763); + }); + + it('metrics at all levels should fail for multiple shifts', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + aggs={aggCount id="2" enabled=true schema="metric"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(result.type).to.be('error'); + }); + }); +} diff --git a/test/interpreter_functional/test_suites/run_pipeline/index.ts b/test/interpreter_functional/test_suites/run_pipeline/index.ts index c33a87a93b903..18d20c97be81e 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/index.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/index.ts @@ -36,5 +36,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid loadTestFile(require.resolve('./tag_cloud')); loadTestFile(require.resolve('./metric')); loadTestFile(require.resolve('./esaggs')); + loadTestFile(require.resolve('./esaggs_timeshift')); }); } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index 3c959693d45ae..a54901a2a2fe1 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -45,7 +45,7 @@ export function FrameLayout(props: FrameLayoutProps) { {props.workspacePanel} - {/* {props.suggestionsPanel} */} + {props.suggestionsPanel}
{localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { const newState = await localState.configurationValidationError?.[0].fixAction?.newState( framePublicAPI diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index a469924fac054..55ab5bee74777 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -522,7 +522,11 @@ export function DimensionEditor(props: DimensionEditorProps) { }, showInPopover: Boolean( operationDefinitionMap[selectedColumn.operationType].shiftable && - selectedColumn.timeShift === undefined + selectedColumn.timeShift === undefined && + (currentIndexPattern.timeFieldName || + Object.values(state.layers[layerId].columns).some( + (col) => col.operationType === 'date_histogram' + )) ), inlineElement: operationDefinitionMap[selectedColumn.operationType].shiftable && diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index f80b12aecabde..4b84c9af31f35 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -34,6 +34,7 @@ import { getFieldByNameFactory } from '../pure_helpers'; import { DimensionEditor } from './dimension_editor'; import { AdvancedOptions } from './advanced_options'; import { Filtering } from './filtering'; +import { TimeShift } from './time_shift'; jest.mock('../loader'); jest.mock('../query_input', () => ({ @@ -1291,6 +1292,199 @@ describe('IndexPatternDimensionEditorPanel', () => { }); }); + describe('time shift', () => { + function getProps(colOverrides: Partial) { + return { + ...defaultProps, + state: getStateWithColumns({ + datecolumn: { + dataType: 'date', + isBucketed: true, + label: '', + customLabel: true, + operationType: 'date_histogram', + sourceField: 'ts', + params: { + interval: '1d', + }, + }, + col2: { + dataType: 'number', + isBucketed: false, + label: 'Count of records', + operationType: 'count', + sourceField: 'Records', + ...colOverrides, + } as IndexPatternColumn, + }), + columnId: 'col2', + }; + } + + it('should not show custom options if time scaling is not available', () => { + const props = getProps({ + operationType: 'terms', + sourceField: 'bytes', + }); + wrapper = shallow( + + ); + expect( + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + ).toHaveLength(0); + }); + + it('should show custom options if time shift is available', () => { + wrapper = shallow(); + expect( + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + ).toHaveLength(1); + }); + + it('should show current time shift if set', () => { + wrapper = mount(); + expect(wrapper.find(TimeShift).find(EuiComboBox).prop('selectedOptions')[0].value).toEqual( + '1d' + ); + }); + + it('should allow to set time shift initially', () => { + const props = getProps({}); + wrapper = shallow(); + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + .prop('onClick')!({} as MouseEvent); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should carry over time shift to other operation if possible', () => { + const props = getProps({ + timeShift: '1d', + sourceField: 'bytes', + operationType: 'sum', + label: 'Sum of bytes per hour', + }); + wrapper = mount(); + wrapper + .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') + .simulate('click'); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1d', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should allow to change time shift', () => { + const props = getProps({ + timeShift: '1d', + }); + wrapper = mount(); + wrapper.find(TimeShift).find(EuiComboBox).prop('onCreateOption')!('1h', []); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1h', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should allow to time shift', () => { + const props = getProps({ + timeShift: '1h', + }); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-time-shift-remove"]') + .find(EuiButtonIcon) + .prop('onClick')!( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + {} as any + ); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: undefined, + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + }); + describe('filtering', () => { function getProps(colOverrides: Partial) { return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 534051ac02c16..6271152f403cc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -141,6 +141,9 @@ export function TimeShift({ const dateHistogramColumn = layer.columnOrder.find( (colId) => layer.columns[colId].operationType === 'date_histogram' ); + if (!dateHistogramColumn && !indexPattern.timeFieldName) { + return null; + } if (dateHistogramColumn && activeData && activeData[layerId] && activeData[layerId]) { const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); if (column) { @@ -169,11 +172,7 @@ export function TimeShift({ } const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); - const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); - const isLocalValueInvalid = Boolean( - parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple - ); + const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid'); function getSelectedOption() { if (!localValue) return []; @@ -207,19 +206,6 @@ export function TimeShift({ helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', })} - error={ - localValueTooSmall - ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { - defaultMessage: - 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', - }) - : localValueNotMultiple - ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { - defaultMessage: - 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', - }) - : undefined - } isInvalid={isLocalValueInvalid} > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index c291c7ab3eac0..d91055d01ade9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -474,6 +474,43 @@ describe('IndexPattern Data Source', () => { expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp', 'another_datefield']); }); + it('should pass time shift parameter to metric agg functions', async () => { + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col2', 'col1'], + columns: { + col1: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: 'Records', + operationType: 'count', + timeShift: '1d', + }, + col2: { + label: 'Date', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'timestamp', + params: { + interval: 'auto', + }, + }, + }, + }, + }, + }; + + const state = enrichBaseState(queryBaseState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + expect((ast.chain[0].arguments.aggs[1] as Ast).chain[0].arguments.timeShift).toEqual(['1d']); + }); + it('should wrap filtered metrics in filtered metric aggregation', async () => { const queryBaseState: IndexPatternBaseState = { currentIndexPatternId: '1', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index c045b1a34d3a5..e784b00c7e10f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect, useState } from 'react'; +import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -22,8 +22,6 @@ import { EuiSwitchEvent, EuiTextColor, } from '@elastic/eui'; -import moment from 'moment'; -import { parseTimeShift } from '../../../../../../../src/plugins/data/common'; import { updateColumnParam } from '../layer_helpers'; import { OperationDefinition, ParamEditorProps } from './index'; import { FieldBasedIndexPatternColumn } from './column_types'; @@ -188,35 +186,12 @@ export const dateHistogramOperation: OperationDefinition< dateRange, data, indexPattern, - activeData, }: ParamEditorProps) { - const [localValue, setLocalValue] = useState(currentColumn?.params.interval); - useEffect(() => { - setLocalValue(currentColumn?.params.interval); - }, [currentColumn?.params.interval]); - - function isIntervalTooSmall(interval: moment.Duration | null) { - if (!interval) { - return false; - } - const timeShifts = Object.values(layer.columns) - .filter((col) => col.timeShift && col.timeShift !== 'previous') - .map((col) => col.timeShift); - return timeShifts.some((shift) => { - if (!shift) return false; - const parsedShift = parseTimeShift(shift); - return ( - typeof parsedShift === 'object' && - parsedShift.asMilliseconds() < interval.asMilliseconds() - ); - }); - } - const field = currentColumn && indexPattern.getFieldByName(currentColumn.sourceField); const intervalIsRestricted = field!.aggregationRestrictions && field!.aggregationRestrictions.date_histogram; - const interval = parseInterval(localValue); + const interval = parseInterval(currentColumn.params.interval); // We force the interval value to 1 if it's empty, since that is the ES behavior, // and the isValidInterval function doesn't handle the empty case properly. Fixing @@ -234,24 +209,13 @@ export const dateHistogramOperation: OperationDefinition< updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); } - const getIntervalAsDuration = (newInterval: typeof interval) => { - const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); - const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; - return search.aggs.parseInterval(value); - }; - - const setInterval = (newInterval: typeof interval, storeLocally: boolean = false) => { + const setInterval = (newInterval: typeof interval) => { const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; - if (storeLocally) { - setLocalValue(value); - } else { - updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); - } + updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); }; - const currentIntervalTooSmall = isIntervalTooSmall(getIntervalAsDuration(interval)); return ( <> {!intervalIsRestricted && ( @@ -273,14 +237,6 @@ export const dateHistogramOperation: OperationDefinition< })} fullWidth display="rowCompressed" - isInvalid={currentIntervalTooSmall} - error={ - currentIntervalTooSmall && - i18n.translate('xpack.lens.indexPattern.intervalTooSmallError', { - defaultMessage: - 'Interval is larger than configured time shift - make sure to keep date histogram interval smaller than time shifts', - }) - } > {intervalIsRestricted ? ( @@ -324,8 +279,7 @@ export const dateHistogramOperation: OperationDefinition< ...interval, unit: e.target.value, }; - const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); - setInterval(newInterval, tooSmall); + setInterval(newInterval); }} isInvalid={!isValid} options={[ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 2e7307f6a2ec4..3e127ffe906e3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -9,7 +9,12 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { shallow, mount } from 'enzyme'; import { EuiFieldNumber, EuiSelect, EuiSwitch, EuiSwitchEvent } from '@elastic/eui'; -import type { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +import type { + IUiSettingsClient, + SavedObjectsClientContract, + HttpSetup, + CoreStart, +} from 'kibana/public'; import type { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../../mocks'; @@ -17,6 +22,7 @@ import { ValuesInput } from './values_input'; import type { TermsIndexPatternColumn } from '.'; import { termsOperation } from '../index'; import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { FramePublicAPI } from '../../../../types'; const uiSettingsMock = {} as IUiSettingsClient; @@ -983,8 +989,8 @@ describe('terms', () => { indexPatternId: '', }; }); - it('returns undefined if sourceField exists in index pattern', () => { - expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual(undefined); + it('returns empty array', () => { + expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([]); }); it('returns error message if the sourceField does not exist in index pattern', () => { layer = { @@ -1000,5 +1006,98 @@ describe('terms', () => { 'Field notExisting was not found', ]); }); + + describe('time shift error', () => { + beforeEach(() => { + layer = { + ...layer, + columnOrder: ['col1', 'col2', 'col3'], + columns: { + ...layer.columns, + col2: { + dataType: 'number', + isBucketed: false, + operationType: 'count', + label: 'Count', + sourceField: 'document', + }, + col3: { + dataType: 'number', + isBucketed: false, + operationType: 'count', + label: 'Count', + sourceField: 'document', + timeShift: '1d', + }, + }, + }; + }); + it('returns error message if two time shifts are used together with terms', () => { + expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ + expect.objectContaining({ + message: + "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + }), + ]); + }); + it('returns fix action which calls field information endpoint and creates a pinned top values', async () => { + const errorMessage = termsOperation.getErrorMessage!(layer, 'col1', indexPattern)![0]; + const fixAction = (typeof errorMessage === 'object' + ? errorMessage.fixAction!.newState + : undefined)!; + const coreMock = ({ + uiSettings: { + get: () => undefined, + }, + http: { + post: jest.fn(() => + Promise.resolve({ + topValues: { + buckets: [ + { + key: 'A', + }, + { + key: 'B', + }, + ], + }, + }) + ), + }, + } as unknown) as CoreStart; + const newLayer = await fixAction(coreMock, ({ + query: { language: 'kuery', query: 'a: b' }, + filters: [], + dateRange: { + fromDate: '2020', + toDate: '2021', + }, + } as unknown) as FramePublicAPI); + expect(newLayer.columns.col1).toEqual( + expect.objectContaining({ + operationType: 'filters', + params: { + filters: [ + { + input: { + language: 'kuery', + query: 'bytes: "A"', + }, + label: 'A', + }, + { + input: { + language: 'kuery', + query: 'bytes: "B"', + }, + label: 'B', + }, + ], + }, + }) + ); + }); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index a3e2a6e567ded..6d020d21d016c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -61,7 +61,7 @@ function getExpressionForLayer( rootDef.input === 'fullReference' && column.timeShift ) { - // inherit filter to all referenced operations + // inherit time shift to all referenced operations column.references.forEach((referenceColumnId) => { const referencedColumn = columns[referenceColumnId]; const referenceDef = operationDefinitionMap[column.operationType]; diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index bfb0aad7177f4..773dfc8c48729 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -36,6 +36,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./persistent_context')); loadTestFile(require.resolve('./colors')); loadTestFile(require.resolve('./chart_data')); + loadTestFile(require.resolve('./time_shift')); loadTestFile(require.resolve('./drag_and_drop')); loadTestFile(require.resolve('./lens_reporting')); loadTestFile(require.resolve('./lens_tagging')); diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts new file mode 100644 index 0000000000000..770ab3cd2c2d6 --- /dev/null +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + + describe('time shift', () => { + it('should able to configure a shifted metric', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'median', + field: 'bytes', + }); + await PageObjects.lens.openDimensionEditor('lnsDatatable_metrics > lns-dimensionTrigger'); + await PageObjects.lens.enableTimeShift(); + await PageObjects.lens.setTimeShift('6h'); + + await PageObjects.lens.waitForVisualization(); + expect(await PageObjects.lens.getDatatableCellText(0, 1)).to.eql('5,994'); + }); + + it('should able to configure a regular metric next to a shifted metric', async () => { + await PageObjects.lens.closeDimensionEditor(); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'average', + field: 'bytes', + }); + + await PageObjects.lens.waitForVisualization(); + + expect(await PageObjects.lens.getDatatableCellText(2, 1)).to.eql('5,994'); + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,722.622'); + }); + + it('should show an error if terms is used and provide a fix action', async () => { + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'terms', + field: 'ip', + }); + + expect(await PageObjects.lens.hasFixAction()).to.be(true); + await PageObjects.lens.useFixAction(); + + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('6,976'); + expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('4,182.5'); + + expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d37..ffa86d3161217 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -350,6 +350,25 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); }, + async enableTimeShift() { + await testSubjects.click('indexPattern-advanced-popover'); + await retry.try(async () => { + await testSubjects.click('indexPattern-time-shift-enable'); + }); + }, + + async setTimeShift(shift: string) { + await comboBox.setCustom('indexPattern-dimension-time-shift', shift); + }, + + async hasFixAction() { + return await testSubjects.exists('errorFixAction'); + }, + + async useFixAction() { + await testSubjects.click('errorFixAction'); + }, + // closes the dimension editor flyout async closeDimensionEditor() { await retry.try(async () => { From 89dd546c85176fa50916960713c07281e28cb29c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 11 May 2021 18:34:02 +0200 Subject: [PATCH 13/31] add back session snapshots --- .../interpreter_functional/snapshots/session/combined_test0.json | 1 + .../interpreter_functional/snapshots/session/combined_test1.json | 1 + .../interpreter_functional/snapshots/session/combined_test2.json | 1 + .../interpreter_functional/snapshots/session/combined_test3.json | 1 + .../snapshots/session/final_output_test.json | 1 + .../snapshots/session/metric_all_data.json | 1 + .../snapshots/session/metric_invalid_data.json | 1 + .../snapshots/session/metric_multi_metric_data.json | 1 + .../snapshots/session/metric_percentage_mode.json | 1 + .../snapshots/session/metric_single_metric_data.json | 1 + .../interpreter_functional/snapshots/session/partial_test_1.json | 1 + .../interpreter_functional/snapshots/session/partial_test_2.json | 1 + .../interpreter_functional/snapshots/session/partial_test_3.json | 1 + .../snapshots/session/step_output_test0.json | 1 + .../snapshots/session/step_output_test1.json | 1 + .../snapshots/session/step_output_test2.json | 1 + .../snapshots/session/step_output_test3.json | 1 + .../snapshots/session/tagcloud_all_data.json | 1 + .../snapshots/session/tagcloud_fontsize.json | 1 + .../snapshots/session/tagcloud_invalid_data.json | 1 + .../snapshots/session/tagcloud_metric_data.json | 1 + .../snapshots/session/tagcloud_options.json | 1 + 22 files changed, 22 insertions(+) create mode 100644 test/interpreter_functional/snapshots/session/combined_test0.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test1.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test2.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test3.json create mode 100644 test/interpreter_functional/snapshots/session/final_output_test.json create mode 100644 test/interpreter_functional/snapshots/session/metric_all_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_invalid_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_multi_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_percentage_mode.json create mode 100644 test/interpreter_functional/snapshots/session/metric_single_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_1.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_2.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_3.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test0.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test1.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test2.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test3.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_all_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_fontsize.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_options.json diff --git a/test/interpreter_functional/snapshots/session/combined_test0.json b/test/interpreter_functional/snapshots/session/combined_test0.json new file mode 100644 index 0000000000000..8f00d72df8ab3 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test0.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test1.json b/test/interpreter_functional/snapshots/session/combined_test1.json new file mode 100644 index 0000000000000..8f00d72df8ab3 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test1.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test2.json b/test/interpreter_functional/snapshots/session/combined_test2.json new file mode 100644 index 0000000000000..4870694e6adbc --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test2.json @@ -0,0 +1 @@ +{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test3.json b/test/interpreter_functional/snapshots/session/combined_test3.json new file mode 100644 index 0000000000000..64b1052552c8f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test3.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/final_output_test.json b/test/interpreter_functional/snapshots/session/final_output_test.json new file mode 100644 index 0000000000000..64b1052552c8f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/final_output_test.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_all_data.json b/test/interpreter_functional/snapshots/session/metric_all_data.json new file mode 100644 index 0000000000000..0e1e5a723373f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_all_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":2,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_invalid_data.json b/test/interpreter_functional/snapshots/session/metric_invalid_data.json new file mode 100644 index 0000000000000..c7b4a0325dc91 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_invalid_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json new file mode 100644 index 0000000000000..fc8622a818dec --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json new file mode 100644 index 0000000000000..95c011f9259b9 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":1000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":true,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json new file mode 100644 index 0000000000000..f4a8cd1f14e18 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_1.json b/test/interpreter_functional/snapshots/session/partial_test_1.json new file mode 100644 index 0000000000000..14c8428c6d432 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_1.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_2.json b/test/interpreter_functional/snapshots/session/partial_test_2.json new file mode 100644 index 0000000000000..64b1052552c8f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_2.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_3.json b/test/interpreter_functional/snapshots/session/partial_test_3.json new file mode 100644 index 0000000000000..e011b69de2022 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_3.json @@ -0,0 +1 @@ +{"as":"region_map_vis","type":"render","value":{"visConfig":{"addTooltip":true,"bucket":{"accessor":0},"colorSchema":"Yellow to Red","isDisplayWarning":true,"legendPosition":"bottomright","mapCenter":[0,0],"mapZoom":2,"metric":{"accessor":1,"format":{"id":"number"}},"outlineWeight":1,"selectedJoinField":{},"selectedLayer":{},"showAllShapes":true,"wms":{}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"region_map"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test0.json b/test/interpreter_functional/snapshots/session/step_output_test0.json new file mode 100644 index 0000000000000..8f00d72df8ab3 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test0.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test1.json b/test/interpreter_functional/snapshots/session/step_output_test1.json new file mode 100644 index 0000000000000..8f00d72df8ab3 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test1.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test2.json b/test/interpreter_functional/snapshots/session/step_output_test2.json new file mode 100644 index 0000000000000..4870694e6adbc --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test2.json @@ -0,0 +1 @@ +{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test3.json b/test/interpreter_functional/snapshots/session/step_output_test3.json new file mode 100644 index 0000000000000..64b1052552c8f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test3.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json new file mode 100644 index 0000000000000..073fca760b9a2 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json new file mode 100644 index 0000000000000..93f8d8a27d233 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json new file mode 100644 index 0000000000000..0c50947beca97 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json new file mode 100644 index 0000000000000..e8c47efdbe622 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_options.json b/test/interpreter_functional/snapshots/session/tagcloud_options.json new file mode 100644 index 0000000000000..38683082975f8 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_options.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file From bcc0cd199c32e677977683d18da45447c50f8a1b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 12 May 2021 09:37:11 +0200 Subject: [PATCH 14/31] fix some bugs --- .../data/common/search/aggs/agg_configs.ts | 5 ++++- .../data/common/search/aggs/metrics/count.ts | 2 +- .../search/aggs/metrics/filtered_metric.ts | 2 +- .../dimension_panel/time_shift.tsx | 2 +- .../operations/layer_helpers.test.ts | 20 +++++++++++++++---- .../indexpattern_datasource/to_expression.ts | 1 + 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index bab7fab5f2248..bdfbc8c90884e 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -457,6 +457,7 @@ export class AggConfigs { } const aggCursor = transformedRawResponse.aggregations!; + const requestAggs = this.getRequestAggs(); const bucketAggs = this.aggs.filter( (agg) => agg.type.type === AggGroupNames.Buckets ) as IBucketAggConfig[]; @@ -480,7 +481,9 @@ export class AggConfigs { return; } else { // a sub-agg - const agg = this.byId(key); + const agg = requestAggs.find( + (requestAgg) => key.substr(0, String(requestAgg.id).length) === requestAgg.id + ); if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); if ( diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index fac1751290f70..7e93fae7e7799 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -35,7 +35,7 @@ export const getCountMetricAgg = () => if (!timeShift) { return bucket.doc_count; } else { - return bucket[`doc_count_${timeShift.asMilliseconds()}`]; + return bucket[`doc_count_${timeShift.asMilliseconds()}`] ?? bucket.doc_count; } }, isScalable() { diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts index aa2417bbf8415..00f47d31b0398 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts @@ -42,7 +42,7 @@ export const getFilteredMetricAgg = () => { getValue(agg, bucket) { const customMetric = agg.getParam('customMetric'); const customBucket = agg.getParam('customBucket'); - return customMetric.getValue(bucket[customBucket.id]); + return bucket && bucket[customBucket.id] && customMetric.getValue(bucket[customBucket.id]); }, getValueBucketPath(agg) { const customBucket = agg.getParam('customBucket'); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 6271152f403cc..4d721ba13468a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -201,7 +201,7 @@ export function TimeShift({ display="columnCompressed" fullWidth label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Time shift', + defaultMessage: 'Shift in time', })} helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index c506e800d6d01..581f9db0d4022 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -2378,7 +2378,10 @@ describe('state_helpers', () => { col1: { operationType: 'average' }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalled(); expect(errors).toHaveLength(1); @@ -2397,7 +2400,10 @@ describe('state_helpers', () => { { operationType: 'testReference', references: [] }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalled(); expect(errors).toHaveLength(1); @@ -2426,7 +2432,10 @@ describe('state_helpers', () => { col1: { operationType: 'testIncompleteReference' }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(savedRef).toHaveBeenCalled(); expect(incompleteRef).not.toHaveBeenCalled(); @@ -2448,7 +2457,10 @@ describe('state_helpers', () => { { operationType: 'testReference', references: [] }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalledWith( { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 6d020d21d016c..c045777e47aa9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -116,6 +116,7 @@ function getExpressionForLayer( }), ]), customMetric: buildExpression({ type: 'expression', chain: [aggAst] }), + timeShift: col.timeShift, } ).toAst(); } From e6f112db48283582fefdbfad64009e3c4ac5c400 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 12 May 2021 10:18:12 +0200 Subject: [PATCH 15/31] fix terms sorting --- src/plugins/data/common/search/aggs/buckets/terms.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 7356d43446358..b9329bcb25af3 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -180,7 +180,11 @@ export const getTermsBucketAgg = () => return; } - if (aggs?.hasTimeShifts() && aggs.timeRange) { + if ( + aggs?.hasTimeShifts() && + Object.keys(aggs?.getTimeShifts()).length > 1 && + aggs.timeRange + ) { const shift = orderAgg.getTimeShift(); orderAgg = aggs.createAggConfig( { From 34a2208bfcd1ad408881e58d136b8d6a06f9bbe2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 17:26:16 +0200 Subject: [PATCH 16/31] fix bug and add tests --- .../data/common/search/aggs/metrics/count.ts | 2 +- .../search/aggs/metrics/moving_avg_fn.ts | 7 + .../run_pipeline/esaggs_timeshift.ts | 177 +++++++++++++++++- .../operations/definitions/cardinality.tsx | 2 +- .../operations/definitions/count.tsx | 2 +- .../operations/definitions/last_value.tsx | 2 +- .../operations/definitions/metrics.tsx | 2 +- .../operations/definitions/percentile.tsx | 2 +- 8 files changed, 188 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index 7e93fae7e7799..fac1751290f70 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -35,7 +35,7 @@ export const getCountMetricAgg = () => if (!timeShift) { return bucket.doc_count; } else { - return bucket[`doc_count_${timeShift.asMilliseconds()}`] ?? bucket.doc_count; + return bucket[`doc_count_${timeShift.asMilliseconds()}`]; } }, isScalable() { diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts index 667c585226a52..63169b4d659bb 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts @@ -94,6 +94,13 @@ export const aggMovingAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts index 7971a7b8ee13a..13cfcde9305f7 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -71,6 +71,40 @@ export default function ({ expect(getCell(result, 0, 2)).to.be(4618); }); + it('shifts single percentile', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggSinglePercentile id="1" enabled=true schema="metric" field="bytes" percentile=95} + aggs={aggSinglePercentile id="2" enabled=true schema="metric" field="bytes" percentile=95 timeShift="1d"} + `; + const result = await expectExpression( + 'esaggs_shift_single_percentile', + expression + ).getResponse(); + // percentile is not stable + expect(getCell(result, 0, 0)).to.be.within(10400, 10500); + expect(getCell(result, 0, 1)).to.be.within(10500, 10600); + }); + + it('shifts multiple percentiles', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggPercentiles id="1" enabled=true schema="metric" field="bytes" percents=5 percents=95} + aggs={aggPercentiles id="2" enabled=true schema="metric" field="bytes" percents=5 percents=95 timeShift="1d"} + `; + const result = await expectExpression( + 'esaggs_shift_multi_percentile', + expression + ).getResponse(); + // percentile is not stable + expect(getCell(result, 0, 0)).to.be.within(300, 400); + expect(getCell(result, 0, 1)).to.be.within(10400, 10500); + expect(getCell(result, 0, 2)).to.be.within(200, 300); + expect(getCell(result, 0, 3)).to.be.within(10500, 10600); + }); + it('shifts date histogram', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -87,6 +121,53 @@ export default function ({ checkShift(result.rows, result.columns); }); + it('shifts filtered metrics', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"} + aggs={aggFilteredMetric + id="2" + customBucket={aggFilter + id="2-filter" + enabled=true + schema="bucket" + filter='{"language":"kuery","query":"geo.src:US"}' + } + customMetric={aggAvg id="3" + field="bytes" + enabled=true + schema="metric" + } + enabled=true + schema="metric" + timeShift="1h" + } + aggs={aggFilteredMetric + id="4" + customBucket={aggFilter + id="4-filter" + enabled=true + schema="bucket" + filter='{"language":"kuery","query":"geo.src:US"}' + } + customMetric={aggAvg id="5" + field="bytes" + enabled=true + schema="metric" + } + enabled=true + schema="metric" + } + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_filtered_metrics', + expression + ).getResponse(); + expect(result.rows.length).to.be(25); + checkShift(result.rows, result.columns); + }); + it('shifts terms', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -123,6 +204,47 @@ export default function ({ ]); }); + it('shifts filters', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggFilters id="1" filters='[{"input":{"query":"geo.src:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.src: \\"CN\\"","language":"kuery"},"label":""}]'} + aggs={aggFilters id="2" filters='[{"input":{"query":"geo.dest:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.dest: \\"CN\\"","language":"kuery"},"label":""}]'} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric" timeShift="2h"} + aggs={aggAvg id="4" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_filters', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 'geo.src:"US" ', + 'col-1-2': 'geo.dest:"US" ', + 'col-2-3': 5956.9, + 'col-3-4': 5956.9, + }, + { + 'col-0-1': 'geo.src:"US" ', + 'col-1-2': 'geo.dest: "CN"', + 'col-2-3': 5127.854838709677, + 'col-3-4': 5085.746031746032, + }, + { + 'col-0-1': 'geo.src: "CN"', + 'col-1-2': 'geo.dest:"US" ', + 'col-2-3': 5648.25, + 'col-3-4': 5643.793650793651, + }, + { + 'col-0-1': 'geo.src: "CN"', + 'col-1-2': 'geo.dest: "CN"', + 'col-2-3': 5842.858823529412, + 'col-3-4': 5842.858823529412, + }, + ]); + }); + it('shifts histogram', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -159,7 +281,7 @@ export default function ({ ]); }); - it('shifts pipeline aggs', async () => { + it('shifts sibling pipeline aggs', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} | esaggs index={indexPatternLoad id='logstash-*'} @@ -167,13 +289,64 @@ export default function ({ aggs={aggBucketSum id="5" enabled=true schema="metric" timeShift="1d" customBucket={aggTerms id="6" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="7" enabled="true" schema="metric"}} `; const result: Datatable = await expectExpression( - 'esaggs_shift_pipeline_aggs', + 'esaggs_shift_sibling_pipeline_aggs', expression ).getResponse(); expect(getCell(result, 0, 0)).to.be(2050); expect(getCell(result, 0, 1)).to.be(2053); }); + it('shifts parent pipeline aggs', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="3h" min_doc_count=0} + aggs={aggMovingAvg id="2" enabled=true schema="metric" metricAgg="custom" window=5 script="MovingFunctions.unweightedAvg(values)" timeShift="3h" customMetric={aggCount id="2-metric" enabled="true" schema="metric"}} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_parent_pipeline_aggs', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 1442791800000, + 'col-1-2': null, + }, + { + 'col-0-1': 1442802600000, + 'col-1-2': 30, + }, + { + 'col-0-1': 1442813400000, + 'col-1-2': 30.5, + }, + { + 'col-0-1': 1442824200000, + 'col-1-2': 69.66666666666667, + }, + { + 'col-0-1': 1442835000000, + 'col-1-2': 198.5, + }, + { + 'col-0-1': 1442845800000, + 'col-1-2': 415.6, + }, + { + 'col-0-1': 1442856600000, + 'col-1-2': 702.2, + }, + { + 'col-0-1': 1442867400000, + 'col-1-2': 859.8, + }, + { + 'col-0-1': 1442878200000, + 'col-1-2': 878.4, + }, + ]); + }); + it('metrics at all levels should work for single shift', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 2e4d083155a94..f53d6136ce2b6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -101,7 +101,7 @@ export const cardinalityOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 0557235a5d9ff..24d34940eccdd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -83,7 +83,7 @@ export const countOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 9acbfa1543307..a2003204f534c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -186,7 +186,7 @@ export const lastValueOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, - timeShift: column.timeShift, + timeShift: column.filter ? undefined : column.timeShift, }).toAst(); }, getErrorMessage: (layer, columnId, indexPattern) => diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 925534afd1251..100a3f8ca57b3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -115,7 +115,7 @@ export const percentileOperation: OperationDefinition Date: Mon, 17 May 2021 19:41:28 +0200 Subject: [PATCH 17/31] address comments --- .../common/search/aggs/agg_configs.test.ts | 8 + .../data/common/search/aggs/agg_configs.ts | 191 +------- .../data/common/search/aggs/metrics/avg_fn.ts | 2 +- .../search/aggs/metrics/bucket_avg_fn.ts | 2 +- .../search/aggs/metrics/bucket_max_fn.ts | 2 +- .../search/aggs/metrics/bucket_min_fn.ts | 2 +- .../search/aggs/metrics/bucket_sum_fn.ts | 2 +- .../search/aggs/metrics/cardinality_fn.ts | 2 +- .../common/search/aggs/metrics/count_fn.ts | 2 +- .../search/aggs/metrics/cumulative_sum_fn.ts | 2 +- .../search/aggs/metrics/derivative_fn.ts | 2 +- .../search/aggs/metrics/filtered_metric_fn.ts | 2 +- .../search/aggs/metrics/geo_bounds_fn.ts | 2 +- .../search/aggs/metrics/geo_centroid_fn.ts | 2 +- .../data/common/search/aggs/metrics/max_fn.ts | 2 +- .../common/search/aggs/metrics/median_fn.ts | 2 +- .../data/common/search/aggs/metrics/min_fn.ts | 2 +- .../search/aggs/metrics/moving_avg_fn.test.ts | 4 + .../search/aggs/metrics/moving_avg_fn.ts | 2 +- .../aggs/metrics/percentile_ranks_fn.ts | 2 +- .../search/aggs/metrics/percentiles_fn.ts | 2 +- .../search/aggs/metrics/serial_diff_fn.ts | 2 +- .../aggs/metrics/single_percentile_fn.ts | 2 +- .../search/aggs/metrics/std_deviation_fn.ts | 2 +- .../data/common/search/aggs/metrics/sum_fn.ts | 2 +- .../common/search/aggs/metrics/top_hit_fn.ts | 2 +- .../search/aggs/utils/parse_time_shift.ts | 2 +- .../common/search/aggs/utils/time_splits.ts | 447 ++++++++++++++++++ .../expressions/esaggs/request_handler.ts | 1 + .../dimension_panel/dimension_panel.test.tsx | 21 +- .../operations/definitions/index.ts | 12 +- .../operations/definitions/terms/index.tsx | 70 ++- .../operations/layer_helpers.ts | 2 +- .../plugins/lens/server/routes/field_stats.ts | 10 +- 34 files changed, 573 insertions(+), 241 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/time_splits.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index d4980447c5468..72ea64791fa5b 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -349,6 +349,14 @@ describe('AggConfigs', () => { params: { field: 'bytes', timeShift: '1d' }, }, ]; + indexPattern.fields.push({ + name: 'timestamp', + type: 'date', + esTypes: ['date'], + aggregatable: true, + filterable: true, + searchable: true, + } as IndexPatternField); const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); ac.timeFields = ['timestamp']; diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index bdfbc8c90884e..6f8a8d38a4a28 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -7,16 +7,10 @@ */ import moment from 'moment'; -import _, { cloneDeep, isArray } from 'lodash'; +import _, { cloneDeep } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; -import { - Aggregate, - Bucket, - FiltersAggregate, - FiltersBucketItem, - MultiBucketAggregate, -} from '@elastic/elasticsearch/api/types'; +import { Aggregate, Bucket } from '@elastic/elasticsearch/api/types'; import { IEsSearchResponse, @@ -31,6 +25,7 @@ import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; import { IBucketAggConfig } from './buckets'; +import { insertTimeShiftSplit, mergeTimeShifts } from './utils/time_splits'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -82,6 +77,7 @@ export class AggConfigs { public indexPattern: IndexPattern; public timeRange?: TimeRange; public timeFields?: string[]; + public forceNow?: Date; public hierarchical?: boolean = false; private readonly typesRegistry: AggTypesRegistryStart; @@ -108,6 +104,10 @@ export class AggConfigs { this.timeFields = timeFields; } + setForceNow(now: Date | undefined) { + this.forceNow = now; + } + setTimeRange(timeRange: TimeRange) { this.timeRange = timeRange; @@ -239,7 +239,7 @@ export class AggConfigs { } if (hasMultipleTimeShifts) { - dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); + dslLvlCursor = insertTimeShiftSplit(this, config, timeShifts, dslLvlCursor); } if (config.type.hasNoDsl) { @@ -284,44 +284,6 @@ export class AggConfigs { return dslTopLvl; } - private insertTimeShiftSplit( - config: AggConfig, - timeShifts: Record, - dslLvlCursor: Record - ) { - if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, this)) { - return dslLvlCursor; - } - if (!this.timeFields || this.timeFields.length < 1) { - throw new Error('Time shift can only be used with configured time field'); - } - if (!this.timeRange) { - throw new Error('Time shift can only be used with configured time range'); - } - const timeRange = this.timeRange; - const filters: Record = {}; - const timeField = this.timeFields[0]; - Object.entries(timeShifts).forEach(([key, shift]) => { - filters[key] = { - range: { - [timeField]: { - // only works if there is a time range - gte: moment(timeRange.from).subtract(shift).toISOString(), - lte: moment(timeRange.to).subtract(shift).toISOString(), - }, - }, - }; - }); - dslLvlCursor.time_offset_split = { - filters: { - filters, - }, - aggs: {}, - }; - - return dslLvlCursor.time_offset_split.aggs; - } - getAll() { return [...this.aggs]; } @@ -447,8 +409,6 @@ export class AggConfigs { if (!this.hasTimeShifts()) { return response; } - const timeShifts = this.getTimeShifts(); - const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); if (!transformedRawResponse.aggregations) { transformedRawResponse.aggregations = { @@ -457,138 +417,7 @@ export class AggConfigs { } const aggCursor = transformedRawResponse.aggregations!; - const requestAggs = this.getRequestAggs(); - const bucketAggs = this.aggs.filter( - (agg) => agg.type.type === AggGroupNames.Buckets - ) as IBucketAggConfig[]; - - const mergeAggLevel = ( - target: GenericBucket, - source: GenericBucket, - shift: moment.Duration, - aggIndex: number - ) => { - Object.entries(source).forEach(([key, val]) => { - // copy over doc count into special key - if (typeof val === 'number' && key === 'doc_count') { - if (shift.asMilliseconds() === 0) { - target.doc_count = val; - } else { - target[`doc_count_${shift.asMilliseconds()}`] = val; - } - } else if (typeof val !== 'object') { - // other meta keys not of interest - return; - } else { - // a sub-agg - const agg = requestAggs.find( - (requestAgg) => key.substr(0, String(requestAgg.id).length) === requestAgg.id - ); - if (agg && agg.type.type === AggGroupNames.Metrics) { - const timeShift = agg.getTimeShift(); - if ( - (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || - (shift.asMilliseconds() === 0 && !timeShift) - ) { - // this is a metric from the current time shift, copy it over - target[key] = source[key]; - } - } else if (agg && agg === bucketAggs[aggIndex]) { - const bucketAgg = agg as IBucketAggConfig; - // expected next bucket sub agg - const subAggregate = val as Aggregate; - const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as - | GenericBucket[] - | Record - | undefined; - if (!target[key]) { - // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate - // which will be filled with shifted data - target[key] = { - buckets: isArray(buckets) ? [] : {}, - }; - } - const baseSubAggregate = target[key] as Aggregate; - // only supported bucket formats in agg configs are array of buckets and record of buckets for filters - const baseBuckets = ('buckets' in baseSubAggregate - ? baseSubAggregate.buckets - : undefined) as GenericBucket[] | Record | undefined; - // merge - if (isArray(buckets) && isArray(baseBuckets)) { - const baseBucketMap: Record = {}; - baseBuckets.forEach((bucket) => { - baseBucketMap[String(bucket.key)] = bucket; - }); - buckets.forEach((bucket) => { - const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); - // if a bucket is missing in the map, create an empty one - if (!baseBucketMap[bucketKey]) { - baseBucketMap[String(bucketKey)] = { - key: bucketKey, - } as GenericBucket; - } - mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); - }); - (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( - baseBucketMap - ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); - } else if (baseBuckets && buckets && !isArray(baseBuckets)) { - Object.entries(buckets).forEach(([bucketKey, bucket]) => { - // if a bucket is missing in the base response, create an empty one - if (!baseBuckets[bucketKey]) { - baseBuckets[bucketKey] = {} as GenericBucket; - } - mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); - }); - } - } - } - }); - }; - const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - const shouldSplit = this.aggs[aggIndex].type.splitForTimeShift(this.aggs[aggIndex], this); - if (shouldSplit) { - // multiple time shifts caused a filters agg in the tree we have to merge - if (hasMultipleTimeShifts && cursor.time_offset_split) { - const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) - .buckets as Record; - const subTree = {}; - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); - - delete cursor.time_offset_split; - Object.assign(cursor, subTree); - } else { - // otherwise we have to "merge" a single level to shift all keys - const [[, shift]] = Object.entries(timeShifts); - const subTree = {}; - mergeAggLevel(subTree, cursor, shift, aggIndex); - Object.assign(cursor, subTree); - } - return; - } - // recurse deeper into the response object - Object.keys(cursor).forEach((subAggId) => { - const subAgg = cursor[subAggId]; - if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { - return; - } - if (isArray(subAgg.buckets)) { - subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); - } else { - Object.values(subAgg.buckets).forEach((bucket) => - transformTimeShift(bucket, aggIndex + 1) - ); - } - }); - }; - transformTimeShift(aggCursor, 0); + mergeTimeShifts(this, aggCursor); return { ...response, rawResponse: transformedRawResponse, diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts index 6b329a8bcfa99..e32de6cd0a83f 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts @@ -66,7 +66,7 @@ export const aggAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts index b9374fb94720a..a980f6ac555a2 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts @@ -81,7 +81,7 @@ export const aggBucketAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts index f50042bc1ca10..0d3e8a5e7f878 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts @@ -81,7 +81,7 @@ export const aggBucketMax = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts index 41bec5c5746c3..3b6c32595909a 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts @@ -81,7 +81,7 @@ export const aggBucketMin = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts index b56dba3b96ba1..ae3502bbc2588 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts @@ -81,7 +81,7 @@ export const aggBucketSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts index fe559b71094ec..89006761407f7 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts @@ -71,7 +71,7 @@ export const aggCardinality = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.ts index 7ca49078ee901..a3a4bcc16a391 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.ts @@ -58,7 +58,7 @@ export const aggCount = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts index 88c1fceb11f72..5cdbcfe857585 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts @@ -85,7 +85,7 @@ export const aggCumulativeSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts index 38985b78cb378..8bfe808aede8e 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts @@ -85,7 +85,7 @@ export const aggDerivative = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts index 7b14fd1c45ed2..0b3d3acd3a603 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts @@ -76,7 +76,7 @@ export const aggFilteredMetric = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts index c7915eb12c5b2..b2cfad1805b9f 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts @@ -71,7 +71,7 @@ export const aggGeoBounds = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts index df1b5182092fd..9215f7afb4c6d 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts @@ -71,7 +71,7 @@ export const aggGeoCentroid = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.ts index f42312a37ecd6..7a1d8ad22fb7e 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.ts @@ -66,7 +66,7 @@ export const aggMax = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.ts index 9145e088d83a9..a9537e1f99ca4 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.ts @@ -71,7 +71,7 @@ export const aggMedian = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.ts index 36b576081ac6e..a97834f310a49 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.ts @@ -66,7 +66,7 @@ export const aggMin = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts index bde90c563afc1..645519a668376 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts @@ -30,6 +30,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, }, "schema": undefined, @@ -59,6 +60,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": "sum", "script": "test", + "timeShift": undefined, "window": 10, }, "schema": undefined, @@ -88,6 +90,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, }, "schema": undefined, @@ -96,6 +99,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, } `); diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts index 63169b4d659bb..1637dad561c37 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts @@ -98,7 +98,7 @@ export const aggMovingAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts index 7bbf8c460682b..60a2882fcec58 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts @@ -78,7 +78,7 @@ export const aggPercentileRanks = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts index 8c9c765d068e2..1a746a86cbcd5 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts @@ -78,7 +78,7 @@ export const aggPercentiles = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts index b481fa56a38ce..8460cb891f1e4 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts @@ -85,7 +85,7 @@ export const aggSerialDiff = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts index b11ed2da8cc76..edf69031c31ac 100644 --- a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts @@ -78,7 +78,7 @@ export const aggSinglePercentile = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts index 7ba0c5b789dbf..c181065d2416e 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts @@ -71,7 +71,7 @@ export const aggStdDeviation = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts index 6e795ada78669..d8e03d28bb12a 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts @@ -66,7 +66,7 @@ export const aggSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts index 9204ca88846c2..bc20f19253eec 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts @@ -98,7 +98,7 @@ export const aggTopHit = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts index e874f489b13f2..256addac91520 100644 --- a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -19,7 +19,7 @@ export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'inv if (val === 'previous') { return 'previous'; } - const [, amount, unit] = val.match(/(\d+)(\w)/) || []; + const [, amount, unit] = val.match(/^(\d+)(\w)$/) || []; const parsedAmount = Number(amount); if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { return 'invalid'; diff --git a/src/plugins/data/common/search/aggs/utils/time_splits.ts b/src/plugins/data/common/search/aggs/utils/time_splits.ts new file mode 100644 index 0000000000000..4ac47efaea347 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/time_splits.ts @@ -0,0 +1,447 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import _, { isArray } from 'lodash'; +import { + Aggregate, + FiltersAggregate, + FiltersBucketItem, + MultiBucketAggregate, +} from '@elastic/elasticsearch/api/types'; + +import { AggGroupNames } from '../agg_groups'; +import { GenericBucket, AggConfigs, getTime, AggConfig } from '../../../../common'; +import { IBucketAggConfig } from '../buckets'; + +/** + * This function will transform an ES response containg a time split (using a filters aggregation before the metrics or date histogram aggregation), + * merging together all branches for the different time ranges into a single response structure which can be tabified into a single table. + * + * If there is just a single time shift, there are no separate branches per time range - in this case only the date histogram keys are shifted by the + * configured amount of time. + * + * To do this, the following steps are taken: + * * Traverse the response tree, tracking the current agg config + * * Once the node which would contain the time split object is found, merge all separate time range buckets into a single layer of buckets of the parent agg + * * Recursively repeat this process for all nested sub-buckets + * + * Example input: + * ``` + * "aggregations" : { + "product" : { + "buckets" : [ + { + "key" : "Product A", + "doc_count" : 512, + "first_year" : { + "doc_count" : 418, + "overall_revenue" : { + "value" : 2163634.0 + } + }, + "time_offset_split" : { + "buckets" : { + "-1y" : { + "doc_count" : 420, + "year" : { + "buckets" : [ + { + "key_as_string" : "2018", + "doc_count" : 81, + "revenue" : { + "value" : 505124.0 + } + }, + { + "key_as_string" : "2019", + "doc_count" : 65, + "revenue" : { + "value" : 363058.0 + } + } + ] + } + }, + "regular" : { + "doc_count" : 418, + "year" : { + "buckets" : [ + { + "key_as_string" : "2019", + "doc_count" : 65, + "revenue" : { + "value" : 363058.0 + } + }, + { + "key_as_string" : "2020", + "doc_count" : 84, + "revenue" : { + "value" : 392924.0 + } + } + ] + } + } + } + } + }, + { + "key" : "Product B", + "doc_count" : 248, + "first_year" : { + "doc_count" : 215, + "overall_revenue" : { + "value" : 1315547.0 + } + }, + "time_offset_split" : { + "buckets" : { + "-1y" : { + "doc_count" : 211, + "year" : { + "buckets" : [ + { + "key_as_string" : "2018", + "key" : 1618963200000, + "doc_count" : 28, + "revenue" : { + "value" : 156543.0 + } + }, + // ... + * ``` + * + * Example output: + * ``` + * "aggregations" : { + "product" : { + "buckets" : [ + { + "key" : "Product A", + "doc_count" : 512, + "first_year" : { + "doc_count" : 418, + "overall_revenue" : { + "value" : 2163634.0 + } + }, + "year" : { + "buckets" : [ + { + "key_as_string" : "2019", + "doc_count" : 81, + "revenue_regular" : { + "value" : 505124.0 + }, + "revenue_-1y" : { + "value" : 302736.0 + } + }, + { + "key_as_string" : "2020", + "doc_count" : 78, + "revenue_regular" : { + "value" : 392924.0 + }, + "revenue_-1y" : { + "value" : 363058.0 + }, + } + // ... + * ``` + * + * + * @param aggConfigs The agg configs instance + * @param aggCursor The root aggregations object from the response which will be mutated in place + */ +export function mergeTimeShifts(aggConfigs: AggConfigs, aggCursor: Record) { + const timeShifts = aggConfigs.getTimeShifts(); + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; + const requestAggs = aggConfigs.getRequestAggs(); + const bucketAggs = aggConfigs.aggs.filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]; + const mergeAggLevel = ( + target: GenericBucket, + source: GenericBucket, + shift: moment.Duration, + aggIndex: number + ) => { + Object.entries(source).forEach(([key, val]) => { + // copy over doc count into special key + if (typeof val === 'number' && key === 'doc_count') { + if (shift.asMilliseconds() === 0) { + target.doc_count = val; + } else { + target[`doc_count_${shift.asMilliseconds()}`] = val; + } + } else if (typeof val !== 'object') { + // other meta keys not of interest + return; + } else { + // a sub-agg + const agg = requestAggs.find((requestAgg) => key.indexOf(requestAgg.id) === 0); + if (agg && agg.type.type === AggGroupNames.Metrics) { + const timeShift = agg.getTimeShift(); + if ( + (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || + (shift.asMilliseconds() === 0 && !timeShift) + ) { + // this is a metric from the current time shift, copy it over + target[key] = source[key]; + } + } else if (agg && agg === bucketAggs[aggIndex]) { + const bucketAgg = agg as IBucketAggConfig; + // expected next bucket sub agg + const subAggregate = val as Aggregate; + const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as + | GenericBucket[] + | Record + | undefined; + if (!target[key]) { + // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate + // which will be filled with shifted data + target[key] = { + buckets: isArray(buckets) ? [] : {}, + }; + } + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters + const baseBuckets = ('buckets' in baseSubAggregate + ? baseSubAggregate.buckets + : undefined) as GenericBucket[] | Record | undefined; + // merge + if (isArray(buckets) && isArray(baseBuckets)) { + const baseBucketMap: Record = {}; + baseBuckets.forEach((bucket) => { + baseBucketMap[String(bucket.key)] = bucket; + }); + buckets.forEach((bucket) => { + const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); + // if a bucket is missing in the map, create an empty one + if (!baseBucketMap[bucketKey]) { + baseBucketMap[String(bucketKey)] = { + key: bucketKey, + } as GenericBucket; + } + mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); + }); + (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( + baseBucketMap + ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); + } else if (baseBuckets && buckets && !isArray(baseBuckets)) { + Object.entries(buckets).forEach(([bucketKey, bucket]) => { + // if a bucket is missing in the base response, create an empty one + if (!baseBuckets[bucketKey]) { + baseBuckets[bucketKey] = {} as GenericBucket; + } + mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); + }); + } + } + } + }); + }; + const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { + const shouldSplit = aggConfigs.aggs[aggIndex].type.splitForTimeShift( + aggConfigs.aggs[aggIndex], + aggConfigs + ); + if (shouldSplit) { + // multiple time shifts caused a filters agg in the tree we have to merge + if (hasMultipleTimeShifts && cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< + string, + FiltersBucketItem + >; + const subTree = {}; + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); + + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + } else { + // otherwise we have to "merge" a single level to shift all keys + const [[, shift]] = Object.entries(timeShifts); + const subTree = {}; + mergeAggLevel(subTree, cursor, shift, aggIndex); + Object.assign(cursor, subTree); + } + return; + } + // recurse deeper into the response object + Object.keys(cursor).forEach((subAggId) => { + const subAgg = cursor[subAggId]; + if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { + return; + } + if (isArray(subAgg.buckets)) { + subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); + } else { + Object.values(subAgg.buckets).forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); + } + }); + }; + transformTimeShift(aggCursor, 0); +} + +/** + * Inserts a filters aggregation into the aggregation tree which splits buckets to fetch data for all time ranges + * configured in metric aggregations. + * + * The current agg config can implement `splitForTimeShift` to force insertion of the time split filters aggregation + * before the dsl of the agg config (date histogram and metrics aggregations do this) + * + * Example aggregation tree without time split: + * ``` + * "aggs": { + "product": { + "terms": { + "field": "product", + "size": 3, + "order": { "overall_revenue": "desc" } + }, + "aggs": { + "overall_revenue": { + "sum": { + "field": "sales" + } + }, + "year": { + "date_histogram": { + "field": "timestamp", + "interval": "year" + }, + "aggs": { + "revenue": { + "sum": { + "field": "sales" + } + } + } + // ... + * ``` + * + * Same aggregation tree with inserted time split: + * ``` + * "aggs": { + "product": { + "terms": { + "field": "product", + "size": 3, + "order": { "first_year>overall_revenue": "desc" } + }, + "aggs": { + "first_year": { + "filter": { + "range": { + "timestamp": { + "gte": "2019", + "lte": "2020" + } + } + }, + "aggs": { + "overall_revenue": { + "sum": { + "field": "sales" + } + } + } + }, + "time_offset_split": { + "filters": { + "filters": { + "regular": { + "range": { + "timestamp": { + "gte": "2019", + "lte": "2020" + } + } + }, + "-1y": { + "range": { + "timestamp": { + "gte": "2018", + "lte": "2019" + } + } + } + } + }, + "aggs": { + "year": { + "date_histogram": { + "field": "timestamp", + "interval": "year" + }, + "aggs": { + "revenue": { + "sum": { + "field": "sales" + } + } + } + } + } + } + } + * ``` + */ +export function insertTimeShiftSplit( + aggConfigs: AggConfigs, + config: AggConfig, + timeShifts: Record, + dslLvlCursor: Record +) { + if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, aggConfigs)) { + return dslLvlCursor; + } + if (!aggConfigs.timeFields || aggConfigs.timeFields.length < 1) { + throw new Error('Time shift can only be used with configured time field'); + } + if (!aggConfigs.timeRange) { + throw new Error('Time shift can only be used with configured time range'); + } + const timeRange = aggConfigs.timeRange; + const filters: Record = {}; + const timeField = aggConfigs.timeFields[0]; + Object.entries(timeShifts).forEach(([key, shift]) => { + const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { + fieldName: timeField, + forceNow: aggConfigs.forceNow, + }); + if (timeFilter) { + filters[key] = { + range: { + [timeField]: { + gte: moment(timeFilter.range[timeField].gte).subtract(shift).toISOString(), + lte: moment(timeFilter.range[timeField].lte).subtract(shift).toISOString(), + }, + }, + }; + } + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; + + return dslLvlCursor.time_offset_split.aggs; +} diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 1eefd71d9e212..61193c52a5e74 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -69,6 +69,7 @@ export const handleRequest = async ({ const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; aggs.setTimeRange(timeRange as TimeRange); + aggs.setForceNow(forceNow); aggs.setTimeFields(allTimeFields); // For now we need to mirror the history of the passed search source, since diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 407e14b1f9623..631e8ce31512d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -34,6 +34,8 @@ import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_ import { getFieldByNameFactory } from '../pure_helpers'; import { Filtering } from './filtering'; import { TimeShift } from './time_shift'; +import { DimensionEditor } from './dimension_editor'; +import { AdvancedOptions } from './advanced_options'; jest.mock('../loader'); jest.mock('../query_input', () => ({ @@ -1333,11 +1335,20 @@ describe('IndexPatternDimensionEditorPanel', () => { }; } - it('should not show custom options if time scaling is not available', () => { - const props = getProps({ - operationType: 'terms', - sourceField: 'bytes', - }); + it('should not show custom options if time shift is not available', () => { + const props = { + ...defaultProps, + state: getStateWithColumns({ + col2: { + dataType: 'number', + isBucketed: false, + label: 'Count of records', + operationType: 'count', + sourceField: 'Records', + } as IndexPatternColumn, + }), + columnId: 'col2', + }; wrapper = shallow( { message: string; fixAction?: { label: string; - newState: (core: CoreStart, frame: FramePublicAPI) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI, + layerId: string + ) => Promise; }; } > @@ -389,7 +393,11 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: (core: CoreStart, frame: FramePublicAPI) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI, + layerId: string + ) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 78b3a13c60e5e..651cf556b09d8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -32,6 +32,7 @@ import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; import type { IndexPatternLayer, IndexPattern } from '../../../types'; +import { defaultLabel } from '../filters'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -80,24 +81,34 @@ function getDisallowedTermsMessage( label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { defaultMessage: 'Pin current top values', }), - newState: async (core: CoreStart, frame: FramePublicAPI) => { - const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await core.http.post( - `/api/lens/index_stats/${indexPattern.id}/field`, - { - body: JSON.stringify({ - fieldName, - dslQuery: esQuery.buildEsQuery( - indexPattern as IIndexPattern, - frame.query, - frame.filters, - esQuery.getEsQueryConfig(core.uiSettings) - ), - fromDate: frame.dateRange.fromDate, - toDate: frame.dateRange.toDate, - }), - } + newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { + const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; + const fieldName = currentColumn.sourceField; + let currentTerms = uniq( + frame.activeData?.[layerId].rows + .map((row) => row[columnId] as string) + .filter((term) => typeof term === 'string' && term !== '__other__') || [] ); + if (currentTerms.length === 0) { + const response: FieldStatsResponse = await core.http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, + size: currentColumn.params.size, + }), + } + ); + currentTerms = response.topValues?.buckets.map(({ key }) => String(key)) || []; + } return { ...layer, columns: { @@ -114,13 +125,24 @@ function getDisallowedTermsMessage( dataType: 'string', operationType: 'filters', params: { - filters: response.topValues?.buckets.map(({ key }) => ({ - input: { - query: `${fieldName}: "${key}"`, - language: 'kuery', - }, - label: String(key), - })), + filters: + currentTerms.length > 0 + ? currentTerms.map((term) => ({ + input: { + query: `${fieldName}: "${term}"`, + language: 'kuery', + }, + label: term, + })) + : [ + { + input: { + query: '*', + language: 'kuery', + }, + label: defaultLabel, + }, + ], }, } as FiltersIndexPatternColumn, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 42de955730ba0..6545b02ad422b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1187,7 +1187,7 @@ export function getErrorMessages( ...state, layers: { ...state.layers, - [layerId]: await errorMessage.fixAction!.newState(core, frame), + [layerId]: await errorMessage.fixAction!.newState(core, frame, layerId), }, }), } diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 6cddd2c60f416..6b7e197a4d561 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -31,6 +31,7 @@ export async function initFieldsRoute(setup: CoreSetup) { fromDate: schema.string(), toDate: schema.string(), fieldName: schema.string(), + size: schema.maybe(schema.number()), }, { unknowns: 'allow' } ), @@ -38,7 +39,7 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = context.core.elasticsearch.client.asCurrentUser; - const { fromDate, toDate, fieldName, dslQuery } = req.body; + const { fromDate, toDate, fieldName, dslQuery, size } = req.body; const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices(); const savedObjectsClient = savedObjects.getScopedClient(req); @@ -112,7 +113,7 @@ export async function initFieldsRoute(setup: CoreSetup) { } return res.ok({ - body: await getStringSamples(search, field), + body: await getStringSamples(search, field, size), }); } catch (e) { if (e instanceof SavedObjectNotFound) { @@ -245,7 +246,8 @@ export async function getNumberHistogram( export async function getStringSamples( aggSearchWithBody: (aggs: Record) => unknown, - field: IFieldType + field: IFieldType, + size = 10 ): Promise { const fieldRef = getFieldRef(field); @@ -257,7 +259,7 @@ export async function getStringSamples( top_values: { terms: { ...fieldRef, - size: 10, + size, }, }, }, From 50844cd75d775916192ce9ed9f9163ab404cc4bc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 19:56:49 +0200 Subject: [PATCH 18/31] fix test --- x-pack/test/functional/apps/lens/time_shift.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts index 770ab3cd2c2d6..9ddd2c9daf58d 100644 --- a/x-pack/test/functional/apps/lens/time_shift.ts +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -59,8 +59,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.hasFixAction()).to.be(true); await PageObjects.lens.useFixAction(); - expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('6,976'); - expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('4,182.5'); + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,541.5'); + expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('3,628'); expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); }); From fc5f171fc6d894f5239ba411bb15caae7f3348fb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 20:52:14 +0200 Subject: [PATCH 19/31] fix types and docs --- ...plugins-data-public.aggconfigs.forcenow.md | 11 ++++++++++ ...a-plugin-plugins-data-public.aggconfigs.md | 2 ++ ...gins-data-public.aggconfigs.setforcenow.md | 22 +++++++++++++++++++ src/plugins/data/public/public.api.md | 4 ++++ .../definitions/terms/terms.test.tsx | 20 ++++++++++------- 5 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md new file mode 100644 index 0000000000000..8040c2939e2e4 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md) + +## AggConfigs.forceNow property + +Signature: + +```typescript +forceNow?: Date; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index ca191fdcae9a5..45333b6767cac 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -22,6 +22,7 @@ export declare class AggConfigs | --- | --- | --- | --- | | [aggs](./kibana-plugin-plugins-data-public.aggconfigs.aggs.md) | | IAggConfig[] | | | [createAggConfig](./kibana-plugin-plugins-data-public.aggconfigs.createaggconfig.md) | | <T extends AggConfig = AggConfig>(params: CreateAggConfigParams, { addToAggConfigs }?: {
addToAggConfigs?: boolean | undefined;
}) => T | | +| [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md) | | Date | | | [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) | | boolean | | | [indexPattern](./kibana-plugin-plugins-data-public.aggconfigs.indexpattern.md) | | IndexPattern | | | [timeFields](./kibana-plugin-plugins-data-public.aggconfigs.timefields.md) | | string[] | | @@ -50,6 +51,7 @@ export declare class AggConfigs | [jsonDataEquals(aggConfigs)](./kibana-plugin-plugins-data-public.aggconfigs.jsondataequals.md) | | Data-by-data comparison of this Aggregation Ignores the non-array indexes | | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | | [postFlightTransform(response)](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) | | | +| [setForceNow(now)](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | | [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md new file mode 100644 index 0000000000000..60a1bfe0872fa --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [setForceNow](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md) + +## AggConfigs.setForceNow() method + +Signature: + +```typescript +setForceNow(now: Date | undefined): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| now | Date | undefined | | + +Returns: + +`void` + diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 14ced4e3346e5..b68b7725bfff5 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -253,6 +253,8 @@ export class AggConfigs { addToAggConfigs?: boolean | undefined; }) => T; // (undocumented) + forceNow?: Date; + // (undocumented) getAll(): AggConfig[]; // (undocumented) getRequestAggById(id: string): AggConfig | undefined; @@ -303,6 +305,8 @@ export class AggConfigs { // (undocumented) postFlightTransform(response: IEsSearchResponse_2): IEsSearchResponse_2; // (undocumented) + setForceNow(now: Date | undefined): void; + // (undocumented) setTimeFields(timeFields: string[] | undefined): void; // (undocumented) setTimeRange(timeRange: TimeRange): void; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 7c751e6db7ef1..3baf004cf01bd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1067,14 +1067,18 @@ describe('terms', () => { ), }, } as unknown) as CoreStart; - const newLayer = await fixAction(coreMock, ({ - query: { language: 'kuery', query: 'a: b' }, - filters: [], - dateRange: { - fromDate: '2020', - toDate: '2021', - }, - } as unknown) as FramePublicAPI); + const newLayer = await fixAction( + coreMock, + ({ + query: { language: 'kuery', query: 'a: b' }, + filters: [], + dateRange: { + fromDate: '2020', + toDate: '2021', + }, + } as unknown) as FramePublicAPI, + 'first' + ); expect(newLayer.columns.col1).toEqual( expect.objectContaining({ operationType: 'filters', From 16b5ee21d860ecbeda07696781a3a408e67a756b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 18 May 2021 12:34:26 +0200 Subject: [PATCH 20/31] increase accepted span for percentiles --- .../test_suites/run_pipeline/esaggs_timeshift.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts index 13cfcde9305f7..c750602f735bd 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -83,8 +83,8 @@ export default function ({ expression ).getResponse(); // percentile is not stable - expect(getCell(result, 0, 0)).to.be.within(10400, 10500); - expect(getCell(result, 0, 1)).to.be.within(10500, 10600); + expect(getCell(result, 0, 0)).to.be.within(10000, 20000); + expect(getCell(result, 0, 1)).to.be.within(10000, 20000); }); it('shifts multiple percentiles', async () => { @@ -99,10 +99,10 @@ export default function ({ expression ).getResponse(); // percentile is not stable - expect(getCell(result, 0, 0)).to.be.within(300, 400); - expect(getCell(result, 0, 1)).to.be.within(10400, 10500); - expect(getCell(result, 0, 2)).to.be.within(200, 300); - expect(getCell(result, 0, 3)).to.be.within(10500, 10600); + expect(getCell(result, 0, 0)).to.be.within(100, 1000); + expect(getCell(result, 0, 1)).to.be.within(10000, 20000); + expect(getCell(result, 0, 2)).to.be.within(100, 1000); + expect(getCell(result, 0, 3)).to.be.within(10000, 20000); }); it('shifts date histogram', async () => { From ef024fe00d2f7bd7fd54f3aa2472879e07682f47 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 19 May 2021 20:00:13 +0200 Subject: [PATCH 21/31] add warnings --- .../workspace_panel_wrapper.tsx | 17 ++- .../dimension_panel/time_shift.tsx | 105 ++++++++++++++- .../indexpattern.test.ts | 124 +++++++++++++++++- .../indexpattern_datasource/indexpattern.tsx | 2 + x-pack/plugins/lens/public/types.ts | 4 + 5 files changed, 246 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx index 85f7601d8fb29..ec12e9e400203 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx @@ -60,9 +60,20 @@ export function WorkspacePanelWrapper({ }, [dispatch, activeVisualization] ); - const warningMessages = - activeVisualization?.getWarningMessages && - activeVisualization.getWarningMessages(visualizationState, framePublicAPI); + const warningMessages: React.ReactNode[] = []; + if (activeVisualization?.getWarningMessages) { + warningMessages.push( + ...(activeVisualization.getWarningMessages(visualizationState, framePublicAPI) || []) + ); + } + Object.entries(datasourceStates).forEach(([datasourceId, datasourceState]) => { + const datasource = datasourceMap[datasourceId]; + if (!datasourceState.isLoading && datasource.getWarningMessages) { + warningMessages.push( + ...(datasource.getWarningMessages(datasourceState.state, framePublicAPI) || []) + ); + } + }); return ( <>
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 4d721ba13468a..02eb3bd00e980 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -8,14 +8,17 @@ import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { EuiComboBox } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; -import { IndexPattern, IndexPatternLayer } from '../types'; +import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; +import { FramePublicAPI } from '../../types'; // to do: get the language from uiSettings export const defaultFilter: Query = { @@ -173,6 +176,8 @@ export function TimeShift({ const parsedLocalValue = localValue && parseTimeShift(localValue); const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid'); + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); function getSelectedOption() { if (!localValue) return []; @@ -206,7 +211,19 @@ export function TimeShift({ helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', })} - isInvalid={isLocalValueInvalid} + error={ + (localValueTooSmall && + i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + })) || + (localValueNotMultiple && + i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { + defaultMessage: + 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + })) + } + isInvalid={Boolean(isLocalValueInvalid || localValueTooSmall || localValueNotMultiple)} > @@ -269,3 +286,87 @@ export function TimeShift({
); } + +export function getTimeShiftWarningMessages( + state: IndexPatternPrivateState, + { activeData }: FramePublicAPI +) { + if (!state) return; + const warningMessages: React.ReactNode[] = []; + Object.entries(state.layers).forEach(([layerId, layer]) => { + let dateHistogramInterval: null | moment.Duration = null; + const dateHistogramColumn = layer.columnOrder.find( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (!dateHistogramColumn) { + return; + } + if (dateHistogramColumn && activeData && activeData[layerId]) { + const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); + if (column) { + dateHistogramInterval = search.aggs.parseInterval( + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' + ); + } + } + if (dateHistogramInterval === null) { + return; + } + const shiftInterval = dateHistogramInterval.asMilliseconds(); + let timeShifts: number[] = []; + const timeShiftMap: Record = {}; + Object.entries(layer.columns).forEach(([columnId, column]) => { + if (column.isBucketed) return; + let duration: number = 0; + if (column.timeShift) { + duration = (parseTimeShift(column.timeShift) as moment.Duration).asMilliseconds(); + } + timeShifts.push(duration); + if (!timeShiftMap[duration]) { + timeShiftMap[duration] = []; + } + timeShiftMap[duration].push(columnId); + }); + timeShifts = uniq(timeShifts); + + if (timeShifts.length < 2) { + return; + } + + timeShifts.forEach((timeShift) => { + if (timeShift === 0) return; + if (timeShift < shiftInterval) { + timeShiftMap[timeShift].forEach((columnId) => { + warningMessages.push( + {layer.columns[columnId].label}, + interval: dateHistogramInterval?.humanize(), + columnTimeShift: layer.columns[columnId].timeShift!, + }} + /> + ); + }); + } else if (!Number.isInteger(timeShift / shiftInterval)) { + timeShiftMap[timeShift].forEach((columnId) => { + warningMessages.push( + {layer.columns[columnId].label}, + interval: dateHistogramInterval?.humanize(), + columnTimeShift: layer.columns[columnId].timeShift!, + }} + /> + ); + }); + } + }); + }); + return warningMessages; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 4ce82aa3a79c0..f0155cc6d59fd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -7,7 +7,7 @@ import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { getIndexPatternDatasource, IndexPatternColumn } from './indexpattern'; -import { DatasourcePublicAPI, Operation, Datasource } from '../types'; +import { DatasourcePublicAPI, Operation, Datasource, FramePublicAPI } from '../types'; import { coreMock } from 'src/core/public/mocks'; import { IndexPatternPersistedState, IndexPatternPrivateState } from './types'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; @@ -18,6 +18,7 @@ import { operationDefinitionMap, getErrorMessages } from './operations'; import { createMockedFullReference } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks'; import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; +import React from 'react'; jest.mock('./loader'); jest.mock('../id_generator'); @@ -1304,6 +1305,127 @@ describe('IndexPattern Data Source', () => { }); }); + describe('#getWarningMessages', () => { + it('should return mismatched time shifts', () => { + const state: IndexPatternPrivateState = { + indexPatternRefs: [], + existingFields: {}, + isFirstExistenceFetch: false, + indexPatterns: expectedIndexPatterns, + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5'], + columns: { + col1: { + operationType: 'date_histogram', + params: { + interval: '12h', + }, + label: '', + dataType: 'date', + isBucketed: true, + sourceField: 'timestamp', + }, + col2: { + operationType: 'count', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col3: { + operationType: 'count', + timeShift: '1h', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col4: { + operationType: 'count', + timeShift: '13h', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col5: { + operationType: 'count', + timeShift: '1w', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + }, + }, + }, + currentIndexPatternId: '1', + }; + const warnings = indexPatternDatasource.getWarningMessages!(state, ({ + activeData: { + first: { + type: 'datatable', + rows: [], + columns: [ + { + id: 'col1', + name: 'col1', + meta: { + type: 'date', + source: 'esaggs', + sourceParams: { + type: 'date_histogram', + params: { + used_interval: '12h', + }, + }, + }, + }, + ], + }, + }, + } as unknown) as FramePublicAPI); + expect(warnings!.length).toBe(2); + expect((warnings![0] as React.ReactElement).props.id).toEqual( + 'xpack.lens.indexPattern.timeShiftSmallWarning' + ); + expect((warnings![1] as React.ReactElement).props.id).toEqual( + 'xpack.lens.indexPattern.timeShiftMultipleWarning' + ); + }); + + it('should prepend each error with its layer number on multi-layer chart', () => { + (getErrorMessages as jest.Mock).mockClear(); + (getErrorMessages as jest.Mock).mockReturnValueOnce(['error 1', 'error 2']); + const state: IndexPatternPrivateState = { + indexPatternRefs: [], + existingFields: {}, + isFirstExistenceFetch: false, + indexPatterns: expectedIndexPatterns, + layers: { + first: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + second: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + }, + currentIndexPatternId: '1', + }; + expect(indexPatternDatasource.getErrorMessages(state)).toEqual([ + { longMessage: 'Layer 1 error: error 1', shortMessage: '' }, + { longMessage: 'Layer 1 error: error 2', shortMessage: '' }, + ]); + expect(getErrorMessages).toHaveBeenCalledTimes(2); + }); + }); + describe('#updateStateOnCloseDimension', () => { it('should not update when there are no incomplete columns', () => { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 0bf91b217c78a..e66df35ea38f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -56,6 +56,7 @@ import { deleteColumn, isReferenced } from './operations'; import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; import { GeoFieldWorkspacePanel } from '../editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel'; import { DraggingIdentifier } from '../drag_drop'; +import { getTimeShiftWarningMessages } from './dimension_panel/time_shift'; export { OperationType, IndexPatternColumn, deleteColumn } from './operations'; @@ -457,6 +458,7 @@ export function getIndexPatternDatasource({ }); return messages.length ? messages : undefined; }, + getWarningMessages: getTimeShiftWarningMessages, checkIntegrity: (state) => { const ids = Object.values(state.layers || {}).map(({ indexPatternId }) => indexPatternId); return ids.filter((id) => !state.indexPatterns[id]); diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 6c799d3fd3d53..cf082eab0f542 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -259,6 +259,10 @@ export interface Datasource { * Check the internal state integrity and returns a list of missing references */ checkIntegrity: (state: T) => string[]; + /** + * The frame calls this function to display warnings about visualization + */ + getWarningMessages?: (state: T, frame: FramePublicAPI) => React.ReactNode[] | undefined; } /** From 047d99f518e4fa1d5db3f70d83e15e1e03e8d357 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 20 May 2021 14:30:52 +0200 Subject: [PATCH 22/31] improve warning --- .../dimension_panel/time_shift.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 02eb3bd00e980..823f41acf40bd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -294,7 +294,7 @@ export function getTimeShiftWarningMessages( if (!state) return; const warningMessages: React.ReactNode[] = []; Object.entries(state.layers).forEach(([layerId, layer]) => { - let dateHistogramInterval: null | moment.Duration = null; + let dateHistogramInterval: null | string = null; const dateHistogramColumn = layer.columnOrder.find( (colId) => layer.columns[colId].operationType === 'date_histogram' ); @@ -304,15 +304,14 @@ export function getTimeShiftWarningMessages( if (dateHistogramColumn && activeData && activeData[layerId]) { const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); if (column) { - dateHistogramInterval = search.aggs.parseInterval( - search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' - ); + dateHistogramInterval = + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || null; } } if (dateHistogramInterval === null) { return; } - const shiftInterval = dateHistogramInterval.asMilliseconds(); + const shiftInterval = search.aggs.parseInterval(dateHistogramInterval)!.asMilliseconds(); let timeShifts: number[] = []; const timeShiftMap: Record = {}; Object.entries(layer.columns).forEach(([columnId, column]) => { @@ -344,8 +343,8 @@ export function getTimeShiftWarningMessages( defaultMessage="{label} uses a time shift of {columnTimeShift} which is smaller than the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift." values={{ label: {layer.columns[columnId].label}, - interval: dateHistogramInterval?.humanize(), - columnTimeShift: layer.columns[columnId].timeShift!, + interval: {dateHistogramInterval}, + columnTimeShift: {layer.columns[columnId].timeShift}, }} /> ); @@ -359,7 +358,7 @@ export function getTimeShiftWarningMessages( defaultMessage="{label} uses a time shift of {columnTimeShift} which is not a multiple of the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift." values={{ label: {layer.columns[columnId].label}, - interval: dateHistogramInterval?.humanize(), + interval: dateHistogramInterval, columnTimeShift: layer.columns[columnId].timeShift!, }} /> From f6620ff60ab0f19e7b6c4838943d3c5fca8ea90a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 11:23:03 +0200 Subject: [PATCH 23/31] review comments --- .../dimension_panel/dimension_editor.tsx | 2 +- .../dimension_panel/time_scaling.tsx | 8 +- .../dimension_panel/time_shift.tsx | 21 ++++- .../definitions/calculations/counter_rate.tsx | 6 +- .../definitions/calculations/differences.tsx | 4 +- .../calculations/moving_average.tsx | 4 +- .../definitions/calculations/utils.ts | 5 +- .../operations/definitions/cardinality.tsx | 28 ++++--- .../operations/definitions/count.tsx | 25 +++++- .../operations/definitions/last_value.tsx | 28 ++++--- .../operations/definitions/metrics.tsx | 11 ++- .../operations/definitions/percentile.tsx | 36 ++++++--- .../operations/time_scale_utils.test.ts | 77 ++++++++++++++++--- .../operations/time_scale_utils.ts | 30 ++++++-- 14 files changed, 218 insertions(+), 67 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 72d8c84b53768..7000117510930 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -489,7 +489,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }, { title: i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Shift in time', + defaultMessage: 'Time shift', }), dataTestSubj: 'indexPattern-time-shift-enable', onClick: () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx index bf5b64bf3d615..61e5da5931e88 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx @@ -27,7 +27,13 @@ export function setTimeScaling( const currentColumn = layer.columns[columnId]; const label = currentColumn.customLabel ? currentColumn.label - : adjustTimeScaleLabelSuffix(currentColumn.label, currentColumn.timeScale, timeScale); + : adjustTimeScaleLabelSuffix( + currentColumn.label, + currentColumn.timeScale, + timeScale, + currentColumn.timeShift, + currentColumn.timeShift + ); return { ...layer, columns: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 823f41acf40bd..514588d534f1b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -15,7 +15,11 @@ import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; -import { IndexPatternColumn, operationDefinitionMap } from '../operations'; +import { + adjustTimeScaleLabelSuffix, + IndexPatternColumn, + operationDefinitionMap, +} from '../operations'; import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { FramePublicAPI } from '../../types'; @@ -31,12 +35,23 @@ export function setTimeShift( layer: IndexPatternLayer, timeShift: string | undefined ) { + const currentColumn = layer.columns[columnId]; + const label = currentColumn.customLabel + ? currentColumn.label + : adjustTimeScaleLabelSuffix( + currentColumn.label, + currentColumn.timeScale, + currentColumn.timeScale, + currentColumn.timeShift, + timeShift + ); return { ...layer, columns: { ...layer.columns, [columnId]: { ...layer.columns[columnId], + label, timeShift, }, }, @@ -206,10 +221,10 @@ export function TimeShift({ display="columnCompressed" fullWidth label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Shift in time', + defaultMessage: 'Time shift', })} helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { - defaultMessage: 'Time shift is specified by a number followed by a time unit', + defaultMessage: 'Enter the time shift number and unit', })} error={ (localValueTooSmall && diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 73d9136d55362..823ec3eb58a92 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -70,7 +70,8 @@ export const counterRateOperation: OperationDefinition< ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName : undefined, - column.timeScale + column.timeScale, + column.timeShift ); }, toExpression: (layer, columnId) => { @@ -84,7 +85,8 @@ export const counterRateOperation: OperationDefinition< metric && 'sourceField' in metric ? indexPattern.getFieldByName(metric.sourceField)?.displayName : undefined, - timeScale + timeScale, + previousColumn?.timeShift ), dataType: 'number', operationType: 'counter_rate', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index 220fcf6dc3c00..7c48b5742b8db 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -66,7 +66,7 @@ export const derivativeOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern, columns) => { - return ofName(columns[column.references[0]]?.label, column.timeScale); + return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'derivative'); @@ -74,7 +74,7 @@ export const derivativeOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { const ref = layer.columns[referenceIds[0]]; return { - label: ofName(ref?.label, previousColumn?.timeScale), + label: ofName(ref?.label, previousColumn?.timeScale, previousColumn?.timeShift), dataType: 'number', operationType: OPERATION_NAME, isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 6351e076faecd..2ee4971fa67f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -80,7 +80,7 @@ export const movingAverageOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern, columns) => { - return ofName(columns[column.references[0]]?.label, column.timeScale); + return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'moving_average', { @@ -94,7 +94,7 @@ export const movingAverageOperation: OperationDefinition< const metric = layer.columns[referenceIds[0]]; const { window = WINDOW_DEFAULT_VALUE } = columnParams; return { - label: ofName(metric?.label, previousColumn?.timeScale), + label: ofName(metric?.label, previousColumn?.timeScale, previousColumn?.timeShift), dataType: 'number', operationType: 'moving_average', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts index 59dbf74c11480..1f4f097c6a7fb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -16,10 +16,11 @@ import { operationDefinitionMap } from '..'; export const buildLabelFunction = (ofName: (name?: string) => string) => ( name?: string, - timeScale?: TimeScaleUnit + timeScale?: TimeScaleUnit, + timeShift?: string ) => { const rawLabel = ofName(name); - return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale); + return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale, undefined, timeShift); }; /** diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index f53d6136ce2b6..c5ea6d21ab485 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -17,6 +17,7 @@ import { getSafeName, getFilter, } from './helpers'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; const supportedTypes = new Set([ 'string', @@ -33,13 +34,19 @@ const SCALE = 'ratio'; const OPERATION_TYPE = 'unique_count'; const IS_BUCKETED = false; -function ofName(name: string) { - return i18n.translate('xpack.lens.indexPattern.cardinalityOf', { - defaultMessage: 'Unique count of {name}', - values: { - name, - }, - }); +function ofName(name: string, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.cardinalityOf', { + defaultMessage: 'Unique count of {name}', + values: { + name, + }, + }), + undefined, + undefined, + undefined, + timeShift + ); } export interface CardinalityIndexPatternColumn @@ -81,10 +88,11 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), + getDefaultLabel: (column, indexPattern) => + ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), buildColumn({ field, previousColumn }, columnParams) { return { - label: ofName(field.displayName), + label: ofName(field.displayName, previousColumn?.timeShift), dataType: 'number', operationType: OPERATION_TYPE, scale: SCALE, @@ -107,7 +115,7 @@ export const cardinalityOperation: OperationDefinition { return { ...oldColumn, - label: ofName(field.displayName), + label: ofName(field.displayName, oldColumn.timeShift), sourceField: field.name, }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 24d34940eccdd..7233b88953eb2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -38,7 +38,13 @@ export const countOperation: OperationDefinition { return { ...oldColumn, - label: adjustTimeScaleLabelSuffix(field.displayName, undefined, oldColumn.timeScale), + label: adjustTimeScaleLabelSuffix( + field.displayName, + undefined, + oldColumn.timeScale, + undefined, + oldColumn.timeShift + ), sourceField: field.name, }; }, @@ -51,10 +57,23 @@ export const countOperation: OperationDefinition adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale), + getDefaultLabel: (column) => + adjustTimeScaleLabelSuffix( + countLabel, + undefined, + column.timeScale, + undefined, + column.timeShift + ), buildColumn({ field, previousColumn }, columnParams) { return { - label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale), + label: adjustTimeScaleLabelSuffix( + countLabel, + undefined, + previousColumn?.timeScale, + undefined, + previousColumn?.timeShift + ), dataType: 'number', operationType: 'count', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index a2003204f534c..dbfc5c7054512 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -21,14 +21,21 @@ import { getSafeName, getFilter, } from './helpers'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; -function ofName(name: string) { - return i18n.translate('xpack.lens.indexPattern.lastValueOf', { - defaultMessage: 'Last value of {name}', - values: { - name, - }, - }); +function ofName(name: string, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.lastValueOf', { + defaultMessage: 'Last value of {name}', + values: { + name, + }, + }), + undefined, + undefined, + undefined, + timeShift + ); } const supportedTypes = new Set(['string', 'boolean', 'number', 'ip']); @@ -96,7 +103,8 @@ export const lastValueOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), + getDefaultLabel: (column, indexPattern) => + ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), input: 'field', onFieldChange: (oldColumn, field) => { const newParams = { ...oldColumn.params }; @@ -107,7 +115,7 @@ export const lastValueOperation: OperationDefinition>({ }) { const labelLookup = (name: string, column?: BaseIndexPatternColumn) => { const label = ofName(name); - if (!optionalTimeScaling) { - return label; - } - return adjustTimeScaleLabelSuffix(label, undefined, column?.timeScale); + return adjustTimeScaleLabelSuffix( + label, + undefined, + optionalTimeScaling ? column?.timeScale : undefined, + undefined, + column?.timeShift + ); }; return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 100a3f8ca57b3..7624c1b9bf83b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -20,6 +20,7 @@ import { getFilter, } from './helpers'; import { FieldBasedIndexPatternColumn } from './column_types'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; export interface PercentileIndexPatternColumn extends FieldBasedIndexPatternColumn { operationType: 'percentile'; @@ -34,12 +35,18 @@ export interface PercentileIndexPatternColumn extends FieldBasedIndexPatternColu }; } -function ofName(name: string, percentile: number) { - return i18n.translate('xpack.lens.indexPattern.percentileOf', { - defaultMessage: - '{percentile, selectordinal, one {#st} two {#nd} few {#rd} other {#th}} percentile of {name}', - values: { name, percentile }, - }); +function ofName(name: string, percentile: number, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.percentileOf', { + defaultMessage: + '{percentile, selectordinal, one {#st} two {#nd} few {#rd} other {#th}} percentile of {name}', + values: { name, percentile }, + }), + undefined, + undefined, + undefined, + timeShift + ); } const DEFAULT_PERCENTILE_VALUE = 95; @@ -75,7 +82,11 @@ export const percentileOperation: OperationDefinition - ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile), + ofName( + getSafeName(column.sourceField, indexPattern), + column.params.percentile, + column.timeShift + ), buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { const existingPercentileParam = previousColumn?.operationType === 'percentile' && @@ -85,7 +96,11 @@ export const percentileOperation: OperationDefinition { return { ...oldColumn, - label: ofName(field.displayName, oldColumn.params.percentile), + label: ofName(field.displayName, oldColumn.params.percentile, oldColumn.timeShift), sourceField: field.name, }; }, @@ -149,7 +164,8 @@ export const percentileOperation: OperationDefinition { describe('adjustTimeScaleLabelSuffix', () => { it('should should remove existing suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc per second', 's', undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined)).toEqual('abc'); + expect( + adjustTimeScaleLabelSuffix('abc per second', 's', undefined, undefined, undefined) + ).toEqual('abc'); + expect( + adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined, undefined, undefined) + ).toEqual('abc'); + expect(adjustTimeScaleLabelSuffix('abc -3d', undefined, undefined, '3d', undefined)).toEqual( + 'abc' + ); + expect( + adjustTimeScaleLabelSuffix('abc per hour -3d', 'h', undefined, '3d', undefined) + ).toEqual('abc'); }); it('should add suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc', undefined, 's')).toEqual('abc per second'); - expect(adjustTimeScaleLabelSuffix('abc', undefined, 'd')).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 's', undefined, undefined)).toEqual( + 'abc per second' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined, undefined, '12h')).toEqual( + 'abc -12h' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 'h', undefined, '12h')).toEqual( + 'abc per hour -12h' + ); + }); + + it('should add and remove at the same time', () => { + expect(adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined, undefined, '1d')).toEqual( + 'abc -1d' + ); + expect(adjustTimeScaleLabelSuffix('abc -1d', undefined, 'h', '1d', undefined)).toEqual( + 'abc per hour' + ); }); it('should change suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc per second', 's', 'd')).toEqual('abc per day'); - expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 's')).toEqual('abc per second'); + expect(adjustTimeScaleLabelSuffix('abc per second', 's', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 's', undefined, undefined)).toEqual( + 'abc per second' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -3h', 'd', 's', '3h', '3h')).toEqual( + 'abc per second -3h' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -3h', 'd', 'd', '3h', '4h')).toEqual( + 'abc per day -4h' + ); }); it('should keep current state', () => { - expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 'd')).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined, undefined, undefined)).toEqual( + 'abc' + ); + expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc -1h', undefined, undefined, '1h', '1h')).toEqual( + 'abc -1h' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -1h', 'd', 'd', '1h', '1h')).toEqual( + 'abc per day -1h' + ); }); it('should not fail on inconsistent input', () => { - expect(adjustTimeScaleLabelSuffix('abc', 's', undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc', 's', 'd')).toEqual('abc per day'); - expect(adjustTimeScaleLabelSuffix('abc per day', 's', undefined)).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', 's', undefined, undefined, undefined)).toEqual( + 'abc' + ); + expect(adjustTimeScaleLabelSuffix('abc', 's', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect( + adjustTimeScaleLabelSuffix('abc per day', 's', undefined, undefined, undefined) + ).toEqual('abc per day'); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts index 07806a32665dd..a0b61060b9f3a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts @@ -12,24 +12,36 @@ import type { IndexPatternColumn } from './definitions'; export const DEFAULT_TIME_SCALE = 's' as TimeScaleUnit; +function getSuffix(scale: TimeScaleUnit | undefined, shift: string | undefined) { + return ( + (shift || scale ? ' ' : '') + + (scale ? unitSuffixesLong[scale] : '') + + (shift && scale ? ' ' : '') + + (shift ? `-${shift}` : '') + ); +} + export function adjustTimeScaleLabelSuffix( oldLabel: string, previousTimeScale: TimeScaleUnit | undefined, - newTimeScale: TimeScaleUnit | undefined + newTimeScale: TimeScaleUnit | undefined, + previousShift: string | undefined, + newShift: string | undefined ) { let cleanedLabel = oldLabel; // remove added suffix if column had a time scale previously - if (previousTimeScale) { - const suffixPosition = oldLabel.lastIndexOf(` ${unitSuffixesLong[previousTimeScale]}`); + if (previousTimeScale || previousShift) { + const suffix = getSuffix(previousTimeScale, previousShift); + const suffixPosition = oldLabel.lastIndexOf(suffix); if (suffixPosition !== -1) { cleanedLabel = oldLabel.substring(0, suffixPosition); } } - if (!newTimeScale) { + if (!newTimeScale && !newShift) { return cleanedLabel; } // add new suffix if column has a time scale now - return `${cleanedLabel} ${unitSuffixesLong[newTimeScale]}`; + return `${cleanedLabel}${getSuffix(newTimeScale, newShift)}`; } export function adjustTimeScaleOnOtherColumnChange( @@ -54,6 +66,12 @@ export function adjustTimeScaleOnOtherColumnChange return { ...column, timeScale: undefined, - label: adjustTimeScaleLabelSuffix(column.label, column.timeScale, undefined), + label: adjustTimeScaleLabelSuffix( + column.label, + column.timeScale, + undefined, + column.timeShift, + column.timeShift + ), }; } From eaa08195c306ed4dc0a31014866005aaebd9f3fc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 17:12:03 +0200 Subject: [PATCH 24/31] adjust wording --- .../operations/definitions/terms/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 651cf556b09d8..02d407fc7360e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -75,11 +75,11 @@ function getDisallowedTermsMessage( return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: - "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', }), fixAction: { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin current top values', + defaultMessage: 'Use filters', }), newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; From b066335608372fb0e4e507622d463bcf40f49e85 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 19:03:02 +0200 Subject: [PATCH 25/31] fix test --- .../operations/definitions/terms/terms.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 3baf004cf01bd..f4b327c175ea6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1037,7 +1037,7 @@ describe('terms', () => { expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ expect.objectContaining({ message: - "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', }), ]); }); From 9c045a7b3af2482ca9427a1dca52d98414b78549 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 26 May 2021 14:52:33 +0200 Subject: [PATCH 26/31] fix previous time shift error checks --- .../dimension_panel/time_shift.tsx | 6 +++++- .../indexpattern_datasource/indexpattern.test.ts | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 514588d534f1b..deaad50d70206 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -333,7 +333,11 @@ export function getTimeShiftWarningMessages( if (column.isBucketed) return; let duration: number = 0; if (column.timeShift) { - duration = (parseTimeShift(column.timeShift) as moment.Duration).asMilliseconds(); + const parsedTimeShift = parseTimeShift(column.timeShift); + if (parsedTimeShift === 'previous' || parsedTimeShift === 'invalid') { + return; + } + duration = parsedTimeShift.asMilliseconds(); } timeShifts.push(duration); if (!timeShiftMap[duration]) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index f0155cc6d59fd..fe212a878db84 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1315,7 +1315,7 @@ describe('IndexPattern Data Source', () => { layers: { first: { indexPatternId: '1', - columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5'], + columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5', 'col6'], columns: { col1: { operationType: 'date_histogram', @@ -1358,6 +1358,14 @@ describe('IndexPattern Data Source', () => { isBucketed: false, sourceField: 'records', }, + col6: { + operationType: 'count', + timeShift: 'previous', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, }, }, }, From 026371fc9840fc76c01f085690392f56beac0d6a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 27 May 2021 11:05:20 +0200 Subject: [PATCH 27/31] add label suffix for cumulative sum --- .../definitions/calculations/cumulative_sum.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 744d362707f14..c4f01e27be886 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -13,11 +13,12 @@ import { getErrorsForDateReference, dateBasedOperationToExpression, hasDateField, + buildLabelFunction, } from './utils'; import { OperationDefinition } from '..'; import { getFormatFromPreviousColumn, getFilter } from '../helpers'; -const ofName = (name?: string) => { +const ofName = buildLabelFunction((name?: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { defaultMessage: 'Cumulative sum of {name}', values: { @@ -28,7 +29,7 @@ const ofName = (name?: string) => { }), }, }); -}; +}); export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn & ReferenceBasedIndexPatternColumn & { @@ -67,7 +68,9 @@ export const cumulativeSumOperation: OperationDefinition< return ofName( ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName - : undefined + : undefined, + undefined, + column.timeShift ); }, toExpression: (layer, columnId) => { @@ -79,7 +82,9 @@ export const cumulativeSumOperation: OperationDefinition< label: ofName( ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName - : undefined + : undefined, + undefined, + previousColumn?.timeShift ), dataType: 'number', operationType: 'cumulative_sum', From 6891e8b015c8578fa04da750ae29272aeeca804b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 27 May 2021 18:55:46 +0200 Subject: [PATCH 28/31] review comments --- .../dimension_panel/time_shift.tsx | 6 ++++-- .../operations/definitions/terms/index.tsx | 9 ++++++--- .../operations/definitions/terms/terms.test.tsx | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index deaad50d70206..538b395bf1624 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -11,7 +11,7 @@ import { EuiComboBox } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; @@ -146,6 +146,7 @@ export function TimeShift({ activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; }) { + const focusSetRef = useRef(false); const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { setLocalValue(selectedColumn.timeShift); @@ -211,7 +212,8 @@ export function TimeShift({ ref={(r) => { if (r && isFocused) { const timeShiftInput = r.querySelector('[data-test-subj="comboBoxSearchInput"]'); - if (timeShiftInput instanceof HTMLInputElement) { + if (!focusSetRef.current && timeShiftInput instanceof HTMLInputElement) { + focusSetRef.current = true; timeShiftInput.focus(); } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 02d407fc7360e..b891a5061a583 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -75,7 +75,7 @@ function getDisallowedTermsMessage( return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: - 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', + 'In a single layer, you are unable to combine metrics with different time shifts and dynamic top values. Use the same time shift value for all metrics, or use filters instead of top values.', }), fixAction: { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { @@ -84,12 +84,15 @@ function getDisallowedTermsMessage( newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; const fieldName = currentColumn.sourceField; + const activeDataFieldNameMatch = + frame.activeData?.[layerId].columns.find(({ id }) => id === columnId)?.meta.field === + fieldName; let currentTerms = uniq( frame.activeData?.[layerId].rows .map((row) => row[columnId] as string) .filter((term) => typeof term === 'string' && term !== '__other__') || [] ); - if (currentTerms.length === 0) { + if (!activeDataFieldNameMatch || currentTerms.length === 0) { const response: FieldStatsResponse = await core.http.post( `/api/lens/index_stats/${indexPattern.id}/field`, { @@ -115,7 +118,7 @@ function getDisallowedTermsMessage( ...layer.columns, [columnId]: { label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { - defaultMessage: 'Pinned top values of {field}', + defaultMessage: 'Filters of {field}', values: { field: fieldName, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index f4b327c175ea6..a0a46556a8459 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1037,7 +1037,7 @@ describe('terms', () => { expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ expect.objectContaining({ message: - 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', + 'In a single layer, you are unable to combine metrics with different time shifts and dynamic top values. Use the same time shift value for all metrics, or use filters instead of top values.', }), ]); }); From 8e9f7248708c1e803a072d21540f049a1cf25e01 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 28 May 2021 09:26:55 +0200 Subject: [PATCH 29/31] fix functional test --- x-pack/test/functional/apps/lens/time_shift.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts index 9ddd2c9daf58d..57c2fc194d0c0 100644 --- a/x-pack/test/functional/apps/lens/time_shift.ts +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -62,7 +62,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,541.5'); expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('3,628'); - expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); + expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Filters of ip'); }); }); } From 3849b044d3e9ee1c3a31a19d0023405c33faa82c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 31 May 2021 09:57:26 +0200 Subject: [PATCH 30/31] review comments --- .../search/aggs/utils/parse_time_shift.ts | 5 +- .../workspace_panel/workspace_panel.tsx | 73 ++++++++++++------- .../dimension_panel/advanced_options.tsx | 11 ++- .../dimension_panel/time_shift.tsx | 6 +- .../operations/definitions/cardinality.tsx | 5 +- .../operations/definitions/count.tsx | 1 + .../operations/definitions/last_value.tsx | 1 + .../operations/definitions/metrics.tsx | 1 + .../operations/definitions/percentile.tsx | 1 + 9 files changed, 67 insertions(+), 37 deletions(-) diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts index 256addac91520..4d8ee0f889173 100644 --- a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -16,10 +16,11 @@ type AllowedUnit = typeof allowedUnits[number]; * Allowed values are the string 'previous' and an integer followed by the units s,m,h,d,w,M,y * */ export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'invalid' => { - if (val === 'previous') { + const trimmedVal = val.trim(); + if (trimmedVal === 'previous') { return 'previous'; } - const [, amount, unit] = val.match(/^(\d+)(\w)$/) || []; + const [, amount, unit] = trimmedVal.match(/^(\d+)(\w)$/) || []; const parsedAmount = Number(amount); if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { return 'invalid'; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 51e7e49a026c2..45abbf120042d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -19,6 +19,7 @@ import { EuiLink, EuiPageContentBody, EuiButton, + EuiSpacer, } from '@elastic/eui'; import { CoreStart, ApplicationStart } from 'kibana/public'; import { @@ -452,6 +453,41 @@ export const VisualizationWrapper = ({ [dispatchLens] ); + function renderFixAction( + validationError: + | { + shortMessage: string; + longMessage: string; + fixAction?: + | { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise } + | undefined; + } + | undefined + ) { + return ( + validationError && + validationError.fixAction && + activeDatasourceId && ( + <> + { + const newState = await validationError.fixAction?.newState(framePublicAPI); + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + datasourceId: activeDatasourceId, + updater: newState, + }); + }} + > + {validationError.fixAction.label} + + + + ) + ); + } + if (localState.configurationValidationError?.length) { let showExtraErrors = null; let showExtraErrorsAction = null; @@ -460,14 +496,17 @@ export const VisualizationWrapper = ({ if (localState.expandError) { showExtraErrors = localState.configurationValidationError .slice(1) - .map(({ longMessage }) => ( -

- {longMessage} -

+ .map((validationError) => ( + <> +

+ {validationError.longMessage} +

+ {renderFixAction(validationError)} + )); } else { showExtraErrorsAction = ( @@ -499,23 +538,7 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].longMessage}

- {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( - { - const newState = await localState.configurationValidationError?.[0].fixAction?.newState( - framePublicAPI - ); - dispatch({ - type: 'UPDATE_DATASOURCE_STATE', - datasourceId: activeDatasourceId, - updater: newState, - }); - }} - > - {localState.configurationValidationError[0].fixAction.label} - - )} + {renderFixAction(localState.configurationValidationError?.[0])} {showExtraErrors} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx index ea5eb14d9c20e..c8676faad0eea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx @@ -20,9 +20,7 @@ export function AdvancedOptions(props: { }) { const [popoverOpen, setPopoverOpen] = useState(false); const popoverOptions = props.options.filter((option) => option.showInPopover); - const inlineOptions = props.options - .filter((option) => option.inlineElement) - .map((option) => React.cloneElement(option.inlineElement!, { key: option.dataTestSubj })); + const inlineOptions = props.options.filter((option) => option.inlineElement); return ( <> @@ -74,7 +72,12 @@ export function AdvancedOptions(props: { {inlineOptions.length > 0 && ( <> - {inlineOptions} + {inlineOptions.map((option, index) => ( + <> + {React.cloneElement(option.inlineElement!, { key: option.dataTestSubj })} + {index !== inlineOptions.length - 1 && } + + ))} )} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 538b395bf1624..0ac02c15b34a5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -35,6 +35,7 @@ export function setTimeShift( layer: IndexPatternLayer, timeShift: string | undefined ) { + const trimmedTimeShift = timeShift?.trim(); const currentColumn = layer.columns[columnId]; const label = currentColumn.customLabel ? currentColumn.label @@ -43,7 +44,7 @@ export function setTimeShift( currentColumn.timeScale, currentColumn.timeScale, currentColumn.timeShift, - timeShift + trimmedTimeShift ); return { ...layer, @@ -52,7 +53,7 @@ export function setTimeShift( [columnId]: { ...layer.columns[columnId], label, - timeShift, + timeShift: trimmedTimeShift, }, }, }; @@ -272,6 +273,7 @@ export function TimeShift({ onChange={(choices) => { if (choices.length === 0) { updateLayer(setTimeShift(columnId, layer, '')); + setLocalValue(''); return; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index c5ea6d21ab485..1911af0a6f679 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -84,10 +84,6 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), buildColumn({ field, previousColumn }, columnParams) { @@ -109,6 +105,7 @@ export const cardinalityOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set timeShift: column.filter ? undefined : column.timeShift, }).toAst(); }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 169fefb4caeb5..aa8f951d46b4f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -130,6 +130,7 @@ export const percentileOperation: OperationDefinition Date: Wed, 2 Jun 2021 13:47:17 +0200 Subject: [PATCH 31/31] fix tests --- .../dimension_panel/dimension_panel.test.tsx | 100 ++++++++---------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 2a07fb0193065..03db6141b917f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -1418,23 +1418,20 @@ describe('IndexPatternDimensionEditorPanel', () => { .dive() .find('[data-test-subj="indexPattern-time-shift-enable"]') .prop('onClick')!({} as MouseEvent); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should carry over time shift to other operation if possible', () => { @@ -1448,23 +1445,20 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '1d', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1d', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to change time shift', () => { @@ -1473,23 +1467,20 @@ describe('IndexPatternDimensionEditorPanel', () => { }); wrapper = mount(); wrapper.find(TimeShift).find(EuiComboBox).prop('onCreateOption')!('1h', []); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '1h', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1h', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to time shift', () => { @@ -1504,23 +1495,20 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: undefined, - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: undefined, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); });