[Reporting] Add output size stats to telemetry metrics (#112037)

* [Reporting] Add output size stats to telemetry metrics

* fix types

* add output_size for each jobtype

* add size metrics for each job type

* use more mock data in unit tests

* clean up test

* update test snapshots

* update telemetry mapping

* SizeMetrics => SizePercentiles

* DocCount interface

* fix tests

* Update get_export_stats.ts

* update snapshots

x-pack/plugins/reporting/server/usage/get_export_stats.test.ts

@@ -11,6 +11,15 @@ import { getExportTypesHandler } from './get_export_type_handler';
 import { FeatureAvailabilityMap } from './types';
 
 let featureMap: FeatureAvailabilityMap;
+const sizesAggResponse = {
+ '1.0': 5093470.0,
+ '5.0': 5093470.0,
+ '25.0': 5093470.0,
+ '50.0': 8514532.0,
+ '75.0': 1.1935594e7,
+ '95.0': 1.1935594e7,
+ '99.0': 1.1935594e7,
+};
 
 beforeEach(() => {
  featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true };
@@ -67,14 +76,19 @@ test('Model of job status and status-by-pdf-app', () => {
 test('Model of jobTypes', () => {
  const result = getExportStats(
  {
- PNG: { available: true, total: 3 },
+ PNG: { available: true, total: 3, sizes: sizesAggResponse },
  printable_pdf: {
  available: true,
  total: 3,
+ sizes: sizesAggResponse,
  app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 },
  layout: { preserve_layout: 3, print: 0 },
  },
- csv_searchsource: { available: true, total: 3 },
+ csv_searchsource: {
+ available: true,
+ total: 3,
+ sizes: sizesAggResponse,
+ },
  },
  featureMap,
  exportTypesHandler
@@ -95,6 +109,15 @@ test('Model of jobTypes', () => {
  "preserve_layout": 0,
  "print": 0,
  },
+ "output_size": Object {
+ "1.0": 5093470,
+ "25.0": 5093470,
+ "5.0": 5093470,
+ "50.0": 8514532,
+ "75.0": 11935594,
+ "95.0": 11935594,
+ "99.0": 11935594,
+ },
  "total": 3,
  }
  `);
@@ -131,6 +154,15 @@ test('Model of jobTypes', () => {
  "preserve_layout": 0,
  "print": 0,
  },
+ "output_size": Object {
+ "1.0": 5093470,
+ "25.0": 5093470,
+ "5.0": 5093470,
+ "50.0": 8514532,
+ "75.0": 11935594,
+ "95.0": 11935594,
+ "99.0": 11935594,
+ },
  "total": 3,
  }
  `);
@@ -149,14 +181,30 @@ test('Model of jobTypes', () => {
  "preserve_layout": 3,
  "print": 0,
  },
+ "output_size": Object {
+ "1.0": 5093470,
+ "25.0": 5093470,
+ "5.0": 5093470,
+ "50.0": 8514532,
+ "75.0": 11935594,
+ "95.0": 11935594,
+ "99.0": 11935594,
+ },
  "total": 3,
  }
  `);
 });
 
 test('PNG counts, provided count of deprecated jobs explicitly', () => {
  const result = getExportStats(
- { PNG: { available: true, total: 15, deprecated: 5 } },
+ {
+ PNG: {
+ available: true,
+ total: 15,
+ deprecated: 5,
+ sizes: sizesAggResponse,
+ },
+ },
  featureMap,
  exportTypesHandler
  );
@@ -175,14 +223,30 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => {
  "preserve_layout": 0,
  "print": 0,
  },
+ "output_size": Object {
+ "1.0": 5093470,
+ "25.0": 5093470,
+ "5.0": 5093470,
+ "50.0": 8514532,
+ "75.0": 11935594,
+ "95.0": 11935594,
+ "99.0": 11935594,
+ },
  "total": 15,
  }
  `);
 });
 
 test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => {
  const result = getExportStats(
- { csv: { available: true, total: 15, deprecated: 0 } },
+ {
+ csv: {
+ available: true,
+ total: 15,
+ deprecated: 0,
+ sizes: sizesAggResponse,
+ },
+ },
  featureMap,
  exportTypesHandler
  );
@@ -201,6 +265,15 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () =>
  "preserve_layout": 0,
  "print": 0,
  },
+ "output_size": Object {
+ "1.0": 5093470,
+ "25.0": 5093470,
+ "5.0": 5093470,
+ "50.0": 8514532,
+ "75.0": 11935594,
+ "95.0": 11935594,
+ "99.0": 11935594,
+ },
  "total": 15,
  }
  `);

x-pack/plugins/reporting/server/usage/get_export_stats.ts

@@ -33,6 +33,7 @@ function getAvailableTotalForFeature(
  available: isAvailable(featureAvailability, typeKey),
  total: jobType.total,
  deprecated,
+ output_size: jobType.sizes,
  app: { ...defaultTotalsForFeature.app, ...jobType.app },
  layout: { ...defaultTotalsForFeature.layout, ...jobType.layout },
  };
@@ -56,6 +57,7 @@ export const getExportStats = (
  _all: rangeAll,
  status: rangeStatus,
  statuses: rangeStatusByApp,
+ output_size: outputSize,
  ...rangeStats
  } = rangeStatsInput;
 
@@ -84,6 +86,7 @@ export const getExportStats = (
  _all: rangeAll || 0,
  status: { completed: 0, failed: 0, ...rangeStatus },
  statuses: rangeStatusByApp,
+ output_size: outputSize,
  } as RangeStats;
 
  return resultStats;

x-pack/plugins/reporting/server/usage/get_reporting_usage.ts

@@ -13,6 +13,7 @@ import type { GetLicense } from './';
 import { getExportStats } from './get_export_stats';
 import { getExportTypesHandler } from './get_export_type_handler';
 import type {
+ AggregationBuckets,
  AggregationResultBuckets,
  AvailableTotal,
  FeatureAvailabilityMap,
@@ -33,6 +34,8 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword';
 const STATUS_TYPES_KEY = 'statusTypes';
 const STATUS_BY_APP_KEY = 'statusByApp';
 const STATUS_TYPES_FIELD = 'status';
+const OUTPUT_SIZES_KEY = 'sizes';
+const OUTPUT_SIZES_FIELD = 'output.size';
 
 const DEFAULT_TERMS_SIZE = 10;
 const PRINTABLE_PDF_JOBTYPE = 'printable_pdf';
@@ -64,13 +67,14 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) =>
  }, {});
 
 function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
- const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY];
+ const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets;
  const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => {
- const { key, doc_count: count, isDeprecated } = bucket;
+ const { key, doc_count: count, isDeprecated, sizes } = bucket;
  const deprecatedCount = isDeprecated?.doc_count;
  const total: Omit<AvailableTotal, 'available'> = {
  total: count,
  deprecated: deprecatedCount,
+ sizes: sizes?.values,
  };
  return { ...accum, [key]: total };
  }, {} as JobTypes);
@@ -97,7 +101,13 @@ function getAggStats(aggs: AggregationResultBuckets): Partial<RangeStats> {
  statusByApp = getAppStatuses(statusAppBuckets);
  }
 
- return { _all: all, status: statusTypes, statuses: statusByApp, ...jobTypes };
+ return {
+ _all: all,
+ status: statusTypes,
+ statuses: statusByApp,
+ output_size: get(aggs[OUTPUT_SIZES_KEY], 'values') ?? undefined,
+ ...jobTypes,
+ };
 }
 
 type RangeStatSets = Partial<RangeStats> & {
@@ -135,7 +145,6 @@ export async function getReportingUsage(
  exportTypesRegistry: ExportTypesRegistry
 ): Promise<ReportingUsageType> {
  const reportingIndex = config.get('index');
-
  const params = {
  index: `${reportingIndex}-*`,
  filterPath: 'aggregations.*.buckets',
@@ -152,28 +161,39 @@ export async function getReportingUsage(
  aggs: {
  [JOB_TYPES_KEY]: {
  terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
- aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } } },
+ aggs: {
+ isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } },
+ [OUTPUT_SIZES_KEY]: {
+ percentiles: { field: OUTPUT_SIZES_FIELD },
+ },
+ },
  },
+
  [STATUS_TYPES_KEY]: { terms: { field: STATUS_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
  [STATUS_BY_APP_KEY]: {
  terms: { field: 'status', size: DEFAULT_TERMS_SIZE },
  aggs: {
  jobTypes: {
  terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE },
  aggs: {
- appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, // NOTE Discover/CSV export is missing the 'meta.objectType' field, so Discover/CSV results are missing for this agg
+ appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
  },
  },
  },
  },
  [OBJECT_TYPES_KEY]: {
  filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
- aggs: { pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
+ aggs: {
+ pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } },
+ },
  },
  [LAYOUT_TYPES_KEY]: {
  filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } },
  aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } },
  },
+ [OUTPUT_SIZES_KEY]: {
+ percentiles: { field: OUTPUT_SIZES_FIELD },
+ },
  },
  },
  },