Skip to content

Commit

Permalink
[Reporting] Add output size stats to telemetry metrics (elastic#112037)
Browse files Browse the repository at this point in the history 
* [Reporting] Add output size stats to telemetry metrics

* fix types

* add output_size for each jobtype

* add size metrics for each job type

* use more mock data in unit tests

* clean up test

* update test snapshots

* update telemetry mapping

* SizeMetrics => SizePercentiles

* DocCount interface

* fix tests

* Update get_export_stats.ts

* update snapshots
  • Loading branch information
@tsullivan
tsullivan committed Sep 23, 2021
1 parent 82dc3ec commit aff884b
Show file tree
Hide file tree
Showing 8 changed files with 1,075 additions and 271 deletions.
530 changes: 462 additions & 68 deletions x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap
View file

Large diffs are not rendered by default.

81 changes: 77 additions & 4 deletions x-pack/plugins/reporting/server/usage/get_export_stats.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,15 @@ import { getExportTypesHandler } from './get_export_type_handler';
import { FeatureAvailabilityMap } from './types';

let featureMap: FeatureAvailabilityMap;
const sizesAggResponse = {
'1.0': 5093470.0,
'5.0': 5093470.0,
'25.0': 5093470.0,
'50.0': 8514532.0,
'75.0': 1.1935594e7,
'95.0': 1.1935594e7,
'99.0': 1.1935594e7,
};

beforeEach(() => {
featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true };
Expand Down Expand Up @@ -67,14 +76,19 @@ test('Model of job status and status-by-pdf-app', () => {
test('Model of jobTypes', () => {
const result = getExportStats(
{
PNG: { available: true, total: 3 },
PNG: { available: true, total: 3, sizes: sizesAggResponse },
printable_pdf: {
available: true,
total: 3,
sizes: sizesAggResponse,
app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 },
layout: { preserve_layout: 3, print: 0 },
},
csv_searchsource: { available: true, total: 3 },
csv_searchsource: {
available: true,
total: 3,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
Expand All @@ -95,6 +109,15 @@ test('Model of jobTypes', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
Expand Down Expand Up @@ -131,6 +154,15 @@ test('Model of jobTypes', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
Expand All @@ -149,14 +181,30 @@ test('Model of jobTypes', () => {
"preserve_layout": 3,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 3,
}
`);
});

test('PNG counts, provided count of deprecated jobs explicitly', () => {
const result = getExportStats(
{ PNG: { available: true, total: 15, deprecated: 5 } },
{
PNG: {
available: true,
total: 15,
deprecated: 5,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
);
Expand All @@ -175,14 +223,30 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => {
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 15,
}
`);
});

test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => {
const result = getExportStats(
{ csv: { available: true, total: 15, deprecated: 0 } },
{
csv: {
available: true,
total: 15,
deprecated: 0,
sizes: sizesAggResponse,
},
},
featureMap,
exportTypesHandler
);
Expand All @@ -201,6 +265,15 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () =>
"preserve_layout": 0,
"print": 0,
},
"output_size": Object {
"1.0": 5093470,
"25.0": 5093470,
"5.0": 5093470,
"50.0": 8514532,
"75.0": 11935594,
"95.0": 11935594,
"99.0": 11935594,
},
"total": 15,
}
`);
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/reporting/server/usage/get_export_stats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ function getAvailableTotalForFeature(
available: isAvailable(featureAvailability, typeKey),
total: jobType.total,
deprecated,
output_size: jobType.sizes,
app: { ...defaultTotalsForFeature.app, ...jobType.app },
layout: { ...defaultTotalsForFeature.layout, ...jobType.layout },
};
Expand All @@ -56,6 +57,7 @@ export const getExportStats = (
_all: rangeAll,
status: rangeStatus,
statuses: rangeStatusByApp,
output_size: outputSize,
...rangeStats
} = rangeStatsInput;

Expand Down Expand Up @@ -84,6 +86,7 @@ export const getExportStats = (
_all: rangeAll || 0,
status: { completed: 0, failed: 0, ...rangeStatus },
statuses: rangeStatusByApp,
output_size: outputSize,
} as RangeStats;

return resultStats;
Expand Down
34 changes: 27 additions & 7 deletions x-pack/plugins/reporting/server/usage/get_reporting_usage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import type { GetLicense } from './';
import { getExportStats } from './get_export_stats';
import { getExportTypesHandler } from './get_export_type_handler';
import type {
AggregationBuckets,
AggregationResultBuckets,
AvailableTotal,
FeatureAvailabilityMap,
Expand All @@ -33,6 +34,8 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword';
const STATUS_TYPES_KEY = 'statusTypes';
const STATUS_BY_APP_KEY = 'statusByApp';
const STATUS_TYPES_FIELD = 'status';
const OUTPUT_SIZES_KEY = 'sizes';
const OUTPUT_SIZES_FIELD = 'output.size';

const DEFAULT_TERMS_SIZE = 10;
const PRINTABLE_PDF_JOBTYPE = 'printable_pdf';
Expand Down Expand Up @@ -64,13 +67,14 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) =>
}, {});

function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY];
const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets;
const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => {
const { key, doc_count: count, isDeprecated } = bucket;
const { key, doc_count: count, isDeprecated, sizes } = bucket;
const deprecatedCount = isDeprecated?.doc_count;
const total: Omit<AvailableTotal, 'available'> = {
total: count,
deprecated: deprecatedCount,
sizes: sizes?.values,
};
return { ...accum, [key]: total };
}, {} as JobTypes);
Expand All @@ -97,7 +101,13 @@ function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
statusByApp = getAppStatuses(statusAppBuckets);
}

return { _all: all, status: statusTypes, statuses: statusByApp, ...jobTypes };
return {
_all: all,
status: statusTypes,
statuses: statusByApp,
output_size: get(aggs[OUTPUT_SIZES_KEY], 'values') ?? undefined,
...jobTypes,
};
}

type RangeStatSets = Partial<RangeStats> & {
Expand Down Expand Up @@ -135,7 +145,6 @@ export async function getReportingUsage(
exportTypesRegistry: ExportTypesRegistry
): Promise<ReportingUsageType> {
const reportingIndex = config.get('index');

const params = {
index: `${reportingIndex}-*`,
filterPath: 'aggregations.*.buckets',
Expand All @@ -152,28 +161,39 @@ export async function getReportingUsage(
aggs: {
[JOB_TYPES_KEY]: {
terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } } },
aggs: {
isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } },
[OUTPUT_SIZES_KEY]: {
percentiles: { field: OUTPUT_SIZES_FIELD },
},
},
},

[STATUS_TYPES_KEY]: { terms: { field: STATUS_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
[STATUS_BY_APP_KEY]: {
terms: { field: 'status', size: DEFAULT_TERMS_SIZE },
aggs: {
jobTypes: {
terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
aggs: {
appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, // NOTE Discover/CSV export is missing the 'meta.objectType' field, so Discover/CSV results are missing for this agg
appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
},
},
},
},
[OBJECT_TYPES_KEY]: {
filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
aggs: { pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
aggs: {
pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
},
},
[LAYOUT_TYPES_KEY]: {
filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
},
[OUTPUT_SIZES_KEY]: {
percentiles: { field: OUTPUT_SIZES_FIELD },
},
},
},
},
Expand Down
Loading

0 comments on commit aff884b

Please sign in to comment.