Skip to content

Commit 80af206

Browse files
anandtiwaryanandtiwary
authored
Zero filling missing bucket in Explore Data (#839)
* refactor: cartesian with null data wip * refactor: zero filling explore time buckets * refactor: fixing test * refactor: self review
1 parent 7c59f3a commit 80af206

File tree

6 files changed

+153
-56
lines changed

6 files changed

+153
-56
lines changed

‎projects/observability/src/shared/dashboard/data/graphql/explore/explore-cartesian-data-source.model.test.ts‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1-
import { ColorService, TimeDuration, TimeUnit } from '@hypertrace/common';
1+
import { ColorService, FixedTimeRange, TimeDuration, TimeUnit } from '@hypertrace/common';
22
import { createModelFactory } from '@hypertrace/dashboards/testing';
33
import {
44
AttributeMetadataType,
55
GraphQlQueryEventService,
6-
GraphQlTimeRange,
76
MetadataService,
87
MetricAggregationType
98
} from '@hypertrace/distributed-tracing';
@@ -29,6 +28,8 @@ import { ExploreCartesianDataSourceModel, ExplorerData } from './explore-cartesi
2928

3029
describe('Explore cartesian data source model', () => {
3130
const testInterval = new TimeDuration(5, TimeUnit.Minute);
31+
const endTime = new Date('2021-05-11T00:35:00.000Z');
32+
const startTime = new Date(endTime.getTime() - 2 * testInterval.toMillis());
3233

3334
const modelFactory = createModelFactory({
3435
providers: [
@@ -74,7 +75,7 @@ describe('Explore cartesian data source model', () => {
7475
beforeEach(() => {
7576
model = modelFactory(TestExploreCartesianDataSourceModel, {
7677
api: {
77-
getTimeRange: jest.fn().mockReturnValue(new GraphQlTimeRange(2, 3))
78+
getTimeRange: () => new FixedTimeRange(startTime, endTime)
7879
}
7980
}).model;
8081
});
@@ -100,14 +101,14 @@ describe('Explore cartesian data source model', () => {
100101
value: 10,
101102
type: AttributeMetadataType.Number
102103
},
103-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
104+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
104105
},
105106
{
106107
'sum(foo)': {
107108
value: 15,
108109
type: AttributeMetadataType.Number
109110
},
110-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
111+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
111112
}
112113
]
113114
},
@@ -122,11 +123,15 @@ describe('Explore cartesian data source model', () => {
122123
type: CartesianSeriesVisualizationType.Line,
123124
data: [
124125
{
125-
timestamp: new Date(0),
126+
timestamp: startTime,
126127
value: 10
127128
},
128129
{
129-
timestamp: new Date(1),
130+
timestamp: new Date('2021-05-11T00:30:00.000Z'),
131+
value: 0
132+
},
133+
{
134+
timestamp: endTime,
130135
value: 15
131136
}
132137
]
@@ -231,7 +236,7 @@ describe('Explore cartesian data source model', () => {
231236
value: 'first',
232237
type: AttributeMetadataType.String
233238
},
234-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
239+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
235240
},
236241
{
237242
'sum(foo)': {
@@ -242,7 +247,7 @@ describe('Explore cartesian data source model', () => {
242247
value: 'first',
243248
type: AttributeMetadataType.String
244249
},
245-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
250+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
246251
},
247252
{
248253
'sum(foo)': {
@@ -253,7 +258,7 @@ describe('Explore cartesian data source model', () => {
253258
value: 'second',
254259
type: AttributeMetadataType.String
255260
},
256-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(0)
261+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: startTime
257262
},
258263
{
259264
'sum(foo)': {
@@ -264,7 +269,7 @@ describe('Explore cartesian data source model', () => {
264269
value: 'second',
265270
type: AttributeMetadataType.String
266271
},
267-
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: new Date(1)
272+
[GQL_EXPLORE_RESULT_INTERVAL_KEY]: endTime
268273
}
269274
]
270275
},
@@ -279,11 +284,15 @@ describe('Explore cartesian data source model', () => {
279284
type: CartesianSeriesVisualizationType.Area,
280285
data: [
281286
{
282-
timestamp: new Date(0),
287+
timestamp: startTime,
283288
value: 10
284289
},
285290
{
286-
timestamp: new Date(1),
291+
timestamp: new Date('2021-05-11T00:30:00.000Z'),
292+
value: 0
293+
},
294+
{
295+
timestamp: endTime,
287296
value: 15
288297
}
289298
]
@@ -294,11 +303,15 @@ describe('Explore cartesian data source model', () => {
294303
type: CartesianSeriesVisualizationType.Area,
295304
data: [
296305
{
297-
timestamp: new Date(0),
306+
timestamp: startTime,
298307
value: 20
299308
},
300309
{
301-
timestamp: new Date(1),
310+
timestamp: new Date('2021-05-11T00:30:00.000Z'),
311+
value: 0
312+
},
313+
{
314+
timestamp: endTime,
302315
value: 25
303316
}
304317
]

‎projects/observability/src/shared/dashboard/data/graphql/explore/explore-cartesian-data-source.model.ts‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { ColorService, forkJoinSafeEmpty, RequireBy, TimeDuration } from '@hypertrace/common';
2-
import { GraphQlDataSourceModel, GraphQlFilter, MetadataService } from '@hypertrace/distributed-tracing';
2+
import {
3+
GraphQlDataSourceModel,
4+
GraphQlFilter,
5+
GraphQlTimeRange,
6+
MetadataService
7+
} from '@hypertrace/distributed-tracing';
38
import { ModelInject } from '@hypertrace/hyperdash-angular';
49
import { isEmpty } from 'lodash-es';
510
import { NEVER, Observable, of } from 'rxjs';
@@ -36,43 +41,58 @@ export abstract class ExploreCartesianDataSourceModel extends GraphQlDataSourceM
3641

3742
protected fetchResults(interval: TimeDuration | 'AUTO'): Observable<CartesianResult<ExplorerData>> {
3843
const requestState = this.buildRequestState(interval);
39-
44+
const timeRange = this.getTimeRangeOrThrow();
4045
if (requestState === undefined) {
4146
return NEVER;
4247
}
4348

4449
return this.query<ExploreGraphQlQueryHandlerService>(inheritedFilters =>
45-
this.buildExploreRequest(requestState, this.getFilters(inheritedFilters))
46-
).pipe(mergeMap(response => this.mapResponseData(requestState, response)));
50+
this.buildExploreRequest(requestState, this.getFilters(inheritedFilters), timeRange)
51+
).pipe(
52+
mergeMap(response =>
53+
this.mapResponseData(requestState, response, requestState.interval as TimeDuration, timeRange)
54+
)
55+
);
4756
}
4857

4958
protected mapResponseData(
5059
requestState: ExploreRequestState,
51-
response: GraphQlExploreResponse
60+
response: GraphQlExploreResponse,
61+
interval: TimeDuration,
62+
timeRange: GraphQlTimeRange
5263
): Observable<CartesianResult<ExplorerData>> {
53-
return this.getAllData(requestState, response).pipe(
64+
return this.getAllData(requestState, response, interval, timeRange).pipe(
5465
map(explorerResults => ({
5566
series: explorerResults,
5667
bands: []
5768
}))
5869
);
5970
}
6071

61-
protected buildExploreRequest(requestState: ExploreRequestState, filters: GraphQlFilter[]): GraphQlExploreRequest {
72+
protected buildExploreRequest(
73+
requestState: ExploreRequestState,
74+
filters: GraphQlFilter[],
75+
timeRange: GraphQlTimeRange
76+
): GraphQlExploreRequest {
6277
return {
6378
requestType: EXPLORE_GQL_REQUEST,
6479
selections: requestState.series.map(series => series.specification),
6580
context: requestState.context,
6681
limit: requestState.resultLimit,
67-
timeRange: this.getTimeRangeOrThrow(),
82+
timeRange: timeRange,
6883
interval: requestState.interval as TimeDuration,
6984
filters: filters,
7085
groupBy: requestState.groupBy
7186
};
7287
}
7388

74-
private getAllData(request: ExploreRequestState, response: GraphQlExploreResponse): Observable<ExplorerSeries[]> {
75-
return this.buildAllSeries(request, new ExploreResult(response));
89+
private getAllData(
90+
request: ExploreRequestState,
91+
response: GraphQlExploreResponse,
92+
interval?: TimeDuration,
93+
timeRange?: GraphQlTimeRange
94+
): Observable<ExplorerSeries[]> {
95+
return this.buildAllSeries(request, new ExploreResult(response, interval, timeRange));
7696
}
7797

7898
protected buildAllSeries(request: ExploreRequestState, result: ExploreResult): Observable<ExplorerSeries[]> {

‎projects/observability/src/shared/dashboard/data/graphql/explore/explore-result.ts‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import { MetricAggregationType } from '@hypertrace/distributed-tracing';
1+
import { TimeDuration } from '@hypertrace/common';
2+
import { GraphQlTimeRange, MetricAggregationType } from '@hypertrace/distributed-tracing';
23
import { groupBy } from 'lodash-es';
34
import { MetricTimeseriesInterval } from '../../../../graphql/model/metric/metric-timeseries';
45
import { ExploreSpecification } from '../../../../graphql/model/schema/specifications/explore-specification';
@@ -15,7 +16,11 @@ export class ExploreResult {
1516

1617
private readonly specBuilder: ExploreSpecificationBuilder = new ExploreSpecificationBuilder();
1718

18-
public constructor(private readonly response: GraphQlExploreResponse) {}
19+
public constructor(
20+
private readonly response: GraphQlExploreResponse,
21+
private readonly interval?: TimeDuration,
22+
private readonly timeRange?: GraphQlTimeRange
23+
) {}
1924

2025
public getTimeSeriesData(metricKey: string, aggregation: MetricAggregationType): MetricTimeseriesInterval[] {
2126
return this.extractTimeseriesForSpec(this.specBuilder.exploreSpecificationForKey(metricKey, aggregation));
@@ -42,7 +47,7 @@ export class ExploreResult {
4247
return new Map(
4348
Object.entries(groupedResults).map(([concatenatedGroupNames, results]) => [
4449
concatenatedGroupNames.split(','),
45-
results.map(result => this.resultToTimeseriesInterval(result, spec))
50+
this.resultsToTimeseriesIntervals(results, spec)
4651
])
4752
);
4853
}
@@ -52,7 +57,7 @@ export class ExploreResult {
5257
}
5358

5459
private extractTimeseriesForSpec(spec: ExploreSpecification): MetricTimeseriesInterval[] {
55-
return this.resultsContainingSpec(spec).map(result => this.resultToTimeseriesInterval(result, spec));
60+
return this.resultsToTimeseriesIntervals(this.resultsContainingSpec(spec), spec);
5661
}
5762

5863
private resultToGroupData(
@@ -72,6 +77,43 @@ export class ExploreResult {
7277
.map(name => (name === ExploreResult.OTHER_SERVER_GROUP_NAME ? ExploreResult.OTHER_UI_GROUP_NAME : name));
7378
}
7479

80+
private resultsToTimeseriesIntervals(
81+
results: GraphQlExploreResult[],
82+
spec: ExploreSpecification
83+
): MetricTimeseriesInterval[] {
84+
if (this.interval !== undefined && this.timeRange !== undefined) {
85+
// This should add missing data to array
86+
87+
// Add all intervals
88+
const buckets = [];
89+
const intervalDuration = this.interval.toMillis();
90+
const startTime = Math.floor(this.timeRange.from.valueOf() / intervalDuration) * intervalDuration;
91+
const endTime = Math.ceil(this.timeRange.to.valueOf() / intervalDuration) * intervalDuration;
92+
93+
for (let timestamp = startTime; timestamp <= endTime; timestamp = timestamp + intervalDuration) {
94+
buckets.push(timestamp);
95+
}
96+
97+
const resultBucketMap: Map<number, MetricTimeseriesInterval> = new Map(
98+
results
99+
.map(result => this.resultToTimeseriesInterval(result, spec))
100+
.map(metric => [metric.timestamp.getTime(), metric])
101+
);
102+
103+
const metrics = buckets.map(
104+
timestamp =>
105+
resultBucketMap.get(timestamp) ?? {
106+
value: 0,
107+
timestamp: new Date(timestamp)
108+
}
109+
);
110+
111+
return metrics;
112+
}
113+
114+
return results.map(result => this.resultToTimeseriesInterval(result, spec));
115+
}
116+
75117
private resultToTimeseriesInterval(
76118
result: GraphQlExploreResult,
77119
spec: ExploreSpecification

0 commit comments

Comments
 (0)