From aff884bc4dbe2556b87201f15c355ef5cd0f40ad Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 23 Sep 2021 16:14:35 -0700 Subject: [PATCH] [Reporting] Add output size stats to telemetry metrics (#112037) * [Reporting] Add output size stats to telemetry metrics * fix types * add output_size for each jobtype * add size metrics for each job type * use more mock data in unit tests * clean up test * update test snapshots * update telemetry mapping * SizeMetrics => SizePercentiles * DocCount interface * fix tests * Update get_export_stats.ts * update snapshots --- .../reporting_usage_collector.test.ts.snap | 530 +++++++++++++++--- .../server/usage/get_export_stats.test.ts | 81 ++- .../server/usage/get_export_stats.ts | 3 + .../server/usage/get_reporting_usage.ts | 34 +- .../usage/reporting_usage_collector.test.ts | 227 ++------ .../plugins/reporting/server/usage/schema.ts | 13 + .../plugins/reporting/server/usage/types.ts | 58 +- .../schema/xpack_plugins.json | 400 +++++++++++++ 8 files changed, 1075 insertions(+), 271 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 12debe5c85d5e..2017ae0be59c7 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -37,6 +37,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -73,6 +96,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -118,6 +164,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -154,6 +223,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -190,6 +282,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -230,6 +345,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -266,6 +404,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -305,6 +466,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -341,6 +525,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -377,10 +584,56 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, }, + "output_size": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "printable_pdf": Object { "app": Object { "canvas workpad": Object { @@ -413,6 +666,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -449,6 +725,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -973,6 +1272,29 @@ Object { }, }, }, + "output_size": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "printable_pdf": Object { "app": Object { "canvas workpad": Object { @@ -1005,6 +1327,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -1041,6 +1386,29 @@ Object { "type": "long", }, }, + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -1620,6 +1988,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 4, }, "csv_searchsource": Object { @@ -1636,6 +2005,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1703,6 +2073,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 4, }, "csv_searchsource": Object { @@ -1719,6 +2090,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1737,6 +2109,7 @@ Object { }, "total": 0, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -1784,6 +2157,7 @@ Object { }, }, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2001,6 +2375,7 @@ Object { }, "total": 0, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2039,6 +2414,7 @@ Object { }, "statuses": Object {}, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2079,7 +2455,7 @@ Object { } `; -exports[`data modeling with normal looking usage data 1`] = ` +exports[`data modeling with sparse data 1`] = ` Object { "PNG": Object { "app": Object { @@ -2095,7 +2471,8 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 3, + "output_size": undefined, + "total": 1, }, "PNGV2": Object { "app": Object { @@ -2113,7 +2490,7 @@ Object { }, "total": 0, }, - "_all": 12, + "_all": 4, "available": true, "browser_type": undefined, "csv": Object { @@ -2124,13 +2501,14 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 0, + "deprecated": 1, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": undefined, + "total": 1, }, "csv_searchsource": Object { "app": Object { @@ -2180,6 +2558,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 1, }, "PNGV2": Object { @@ -2198,7 +2577,7 @@ Object { }, "total": 0, }, - "_all": 1, + "_all": 4, "csv": Object { "app": Object { "canvas workpad": 0, @@ -2207,13 +2586,14 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 0, + "deprecated": 1, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": undefined, + "total": 1, }, "csv_searchsource": Object { "app": Object { @@ -2247,10 +2627,11 @@ Object { }, "total": 0, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { - "canvas workpad": 0, - "dashboard": 0, + "canvas workpad": 1, + "dashboard": 1, "search": 0, "visualization": 0, }, @@ -2258,10 +2639,11 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 0, + "preserve_layout": 2, "print": 0, }, - "total": 0, + "output_size": undefined, + "total": 2, }, "printable_pdf_v2": Object { "app": Object { @@ -2280,33 +2662,39 @@ Object { "total": 0, }, "status": Object { - "completed": 0, - "completed_with_warnings": 1, + "completed": 4, "failed": 0, }, "statuses": Object { - "completed_with_warnings": Object { + "completed": Object { "PNG": Object { "dashboard": 1, }, + "csv": Object {}, + "printable_pdf": Object { + "canvas workpad": 1, + "dashboard": 1, + }, }, }, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { - "canvas workpad": 6, - "dashboard": 0, + "canvas workpad": 1, + "dashboard": 1, "search": 0, - "visualization": 3, + "visualization": 0, }, "available": true, "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 9, + "preserve_layout": 2, "print": 0, }, - "total": 9, + "output_size": undefined, + "total": 2, }, "printable_pdf_v2": Object { "app": Object { @@ -2325,27 +2713,17 @@ Object { "total": 0, }, "status": Object { - "completed": 10, - "completed_with_warnings": 1, - "failed": 1, + "completed": 4, + "failed": 0, }, "statuses": Object { "completed": Object { - "PNG": Object { - "visualization": 1, - }, - "printable_pdf": Object { - "canvas workpad": 6, - "visualization": 3, - }, - }, - "completed_with_warnings": Object { "PNG": Object { "dashboard": 1, }, - }, - "failed": Object { - "PNG": Object { + "csv": Object {}, + "printable_pdf": Object { + "canvas workpad": 1, "dashboard": 1, }, }, @@ -2353,7 +2731,7 @@ Object { } `; -exports[`data modeling with sparse data 1`] = ` +exports[`data modeling with usage data from the reporting/archived_reports es archive 1`] = ` Object { "PNG": Object { "app": Object { @@ -2369,6 +2747,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 1, }, "PNGV2": Object { @@ -2387,7 +2766,7 @@ Object { }, "total": 0, }, - "_all": 4, + "_all": 11, "available": true, "browser_type": undefined, "csv": Object { @@ -2404,6 +2783,7 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": undefined, "total": 1, }, "csv_searchsource": Object { @@ -2420,7 +2800,8 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": undefined, + "total": 3, }, "csv_searchsource_immediate": Object { "app": Object { @@ -2454,7 +2835,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 1, + "total": 0, }, "PNGV2": Object { "app": Object { @@ -2472,7 +2853,7 @@ Object { }, "total": 0, }, - "_all": 4, + "_all": 0, "csv": Object { "app": Object { "canvas workpad": 0, @@ -2481,13 +2862,13 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 1, + "deprecated": 0, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 1, + "total": 0, }, "csv_searchsource": Object { "app": Object { @@ -2521,10 +2902,11 @@ Object { }, "total": 0, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { - "canvas workpad": 1, - "dashboard": 1, + "canvas workpad": 0, + "dashboard": 0, "search": 0, "visualization": 0, }, @@ -2532,10 +2914,10 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 2, + "preserve_layout": 0, "print": 0, }, - "total": 2, + "total": 0, }, "printable_pdf_v2": Object { "app": Object { @@ -2554,26 +2936,16 @@ Object { "total": 0, }, "status": Object { - "completed": 4, + "completed": 0, "failed": 0, }, - "statuses": Object { - "completed": Object { - "PNG": Object { - "dashboard": 1, - }, - "csv": Object {}, - "printable_pdf": Object { - "canvas workpad": 1, - "dashboard": 1, - }, - }, - }, + "statuses": Object {}, }, + "output_size": undefined, "printable_pdf": Object { "app": Object { - "canvas workpad": 1, - "dashboard": 1, + "canvas workpad": 0, + "dashboard": 6, "search": 0, "visualization": 0, }, @@ -2581,10 +2953,11 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 2, - "print": 0, + "preserve_layout": 5, + "print": 1, }, - "total": 2, + "output_size": undefined, + "total": 6, }, "printable_pdf_v2": Object { "app": Object { @@ -2603,17 +2976,38 @@ Object { "total": 0, }, "status": Object { - "completed": 4, - "failed": 0, + "completed": 6, + "completed_with_warnings": 2, + "failed": 2, + "pending": 1, }, "statuses": Object { "completed": Object { + "csv": Object { + "search": 1, + }, + "csv_searchsource": Object { + "search": 3, + }, + "printable_pdf": Object { + "dashboard": 2, + }, + }, + "completed_with_warnings": Object { "PNG": Object { "dashboard": 1, }, - "csv": Object {}, "printable_pdf": Object { - "canvas workpad": 1, + "dashboard": 1, + }, + }, + "failed": Object { + "printable_pdf": Object { + "dashboard": 2, + }, + }, + "pending": Object { + "printable_pdf": Object { "dashboard": 1, }, }, diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index 782f2e910038e..f74e176e6f21d 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -11,6 +11,15 @@ import { getExportTypesHandler } from './get_export_type_handler'; import { FeatureAvailabilityMap } from './types'; let featureMap: FeatureAvailabilityMap; +const sizesAggResponse = { + '1.0': 5093470.0, + '5.0': 5093470.0, + '25.0': 5093470.0, + '50.0': 8514532.0, + '75.0': 1.1935594e7, + '95.0': 1.1935594e7, + '99.0': 1.1935594e7, +}; beforeEach(() => { featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true }; @@ -67,14 +76,19 @@ test('Model of job status and status-by-pdf-app', () => { test('Model of jobTypes', () => { const result = getExportStats( { - PNG: { available: true, total: 3 }, + PNG: { available: true, total: 3, sizes: sizesAggResponse }, printable_pdf: { available: true, total: 3, + sizes: sizesAggResponse, app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 }, layout: { preserve_layout: 3, print: 0 }, }, - csv_searchsource: { available: true, total: 3 }, + csv_searchsource: { + available: true, + total: 3, + sizes: sizesAggResponse, + }, }, featureMap, exportTypesHandler @@ -95,6 +109,15 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, + }, "total": 3, } `); @@ -131,6 +154,15 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, + }, "total": 3, } `); @@ -149,6 +181,15 @@ test('Model of jobTypes', () => { "preserve_layout": 3, "print": 0, }, + "output_size": Object { + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, + }, "total": 3, } `); @@ -156,7 +197,14 @@ test('Model of jobTypes', () => { test('PNG counts, provided count of deprecated jobs explicitly', () => { const result = getExportStats( - { PNG: { available: true, total: 15, deprecated: 5 } }, + { + PNG: { + available: true, + total: 15, + deprecated: 5, + sizes: sizesAggResponse, + }, + }, featureMap, exportTypesHandler ); @@ -175,6 +223,15 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, + }, "total": 15, } `); @@ -182,7 +239,14 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => { const result = getExportStats( - { csv: { available: true, total: 15, deprecated: 0 } }, + { + csv: { + available: true, + total: 15, + deprecated: 0, + sizes: sizesAggResponse, + }, + }, featureMap, exportTypesHandler ); @@ -201,6 +265,15 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, + }, "total": 15, } `); diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index ffdb6cdc290d7..72c09f08017a1 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -33,6 +33,7 @@ function getAvailableTotalForFeature( available: isAvailable(featureAvailability, typeKey), total: jobType.total, deprecated, + output_size: jobType.sizes, app: { ...defaultTotalsForFeature.app, ...jobType.app }, layout: { ...defaultTotalsForFeature.layout, ...jobType.layout }, }; @@ -56,6 +57,7 @@ export const getExportStats = ( _all: rangeAll, status: rangeStatus, statuses: rangeStatusByApp, + output_size: outputSize, ...rangeStats } = rangeStatsInput; @@ -84,6 +86,7 @@ export const getExportStats = ( _all: rangeAll || 0, status: { completed: 0, failed: 0, ...rangeStatus }, statuses: rangeStatusByApp, + output_size: outputSize, } as RangeStats; return resultStats; diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 9aba7841162c2..9a452943ff699 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -13,6 +13,7 @@ import type { GetLicense } from './'; import { getExportStats } from './get_export_stats'; import { getExportTypesHandler } from './get_export_type_handler'; import type { + AggregationBuckets, AggregationResultBuckets, AvailableTotal, FeatureAvailabilityMap, @@ -33,6 +34,8 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword'; const STATUS_TYPES_KEY = 'statusTypes'; const STATUS_BY_APP_KEY = 'statusByApp'; const STATUS_TYPES_FIELD = 'status'; +const OUTPUT_SIZES_KEY = 'sizes'; +const OUTPUT_SIZES_FIELD = 'output.size'; const DEFAULT_TERMS_SIZE = 10; const PRINTABLE_PDF_JOBTYPE = 'printable_pdf'; @@ -64,13 +67,14 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) => }, {}); function getAggStats(aggs: AggregationResultBuckets): Partial { - const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY]; + const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets; const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => { - const { key, doc_count: count, isDeprecated } = bucket; + const { key, doc_count: count, isDeprecated, sizes } = bucket; const deprecatedCount = isDeprecated?.doc_count; const total: Omit = { total: count, deprecated: deprecatedCount, + sizes: sizes?.values, }; return { ...accum, [key]: total }; }, {} as JobTypes); @@ -97,7 +101,13 @@ function getAggStats(aggs: AggregationResultBuckets): Partial { statusByApp = getAppStatuses(statusAppBuckets); } - return { _all: all, status: statusTypes, statuses: statusByApp, ...jobTypes }; + return { + _all: all, + status: statusTypes, + statuses: statusByApp, + output_size: get(aggs[OUTPUT_SIZES_KEY], 'values') ?? undefined, + ...jobTypes, + }; } type RangeStatSets = Partial & { @@ -135,7 +145,6 @@ export async function getReportingUsage( exportTypesRegistry: ExportTypesRegistry ): Promise { const reportingIndex = config.get('index'); - const params = { index: `${reportingIndex}-*`, filterPath: 'aggregations.*.buckets', @@ -152,8 +161,14 @@ export async function getReportingUsage( aggs: { [JOB_TYPES_KEY]: { terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE }, - aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } } }, + aggs: { + isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } }, + [OUTPUT_SIZES_KEY]: { + percentiles: { field: OUTPUT_SIZES_FIELD }, + }, + }, }, + [STATUS_TYPES_KEY]: { terms: { field: STATUS_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, [STATUS_BY_APP_KEY]: { terms: { field: 'status', size: DEFAULT_TERMS_SIZE }, @@ -161,19 +176,24 @@ export async function getReportingUsage( jobTypes: { terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE }, aggs: { - appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, // NOTE Discover/CSV export is missing the 'meta.objectType' field, so Discover/CSV results are missing for this agg + appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, }, }, }, }, [OBJECT_TYPES_KEY]: { filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, - aggs: { pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, + aggs: { + pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, + }, }, [LAYOUT_TYPES_KEY]: { filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, }, + [OUTPUT_SIZES_KEY]: { + percentiles: { field: OUTPUT_SIZES_FIELD }, + }, }, }, }, diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index e69e56d6272d5..447085810cfd0 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -18,7 +18,6 @@ import { getReportingUsageCollector, registerReportingUsageCollector, } from './reporting_usage_collector'; -import { SearchResponse } from './types'; const exportTypesRegistry = getExportTypesRegistry(); @@ -190,7 +189,7 @@ describe('data modeling', () => { beforeAll(async () => { mockCore = await createMockReportingCore(createMockConfigSchema()); }); - test('with normal looking usage data', async () => { + test('with usage data from the reporting/archived_reports es archive', async () => { const plugins = getPluginsMock(); const collector = getReportingUsageCollector( mockCore, @@ -202,39 +201,37 @@ describe('data modeling', () => { } ); collectorFetchContext = getMockFetchClients( - getResponseMock( - { + getResponseMock({ aggregations: { ranges: { + meta: {}, buckets: { all: { - doc_count: 12, - jobTypes: { buckets: [ { doc_count: 9, key: 'printable_pdf' }, { doc_count: 3, key: 'PNG' }, ], }, - layoutTypes: { doc_count: 9, pdf: { buckets: [{ doc_count: 9, key: 'preserve_layout' }] }, }, - objectTypes: { doc_count: 9, pdf: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, }, - statusByApp: { buckets: [ { doc_count: 10, jobTypes: { buckets: [ { appNames: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, doc_count: 9, key: 'printable_pdf', }, { appNames: { buckets: [{ doc_count: 1, key: 'visualization' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'failed', }, ], }, - statusTypes: { buckets: [ { doc_count: 10, key: 'completed' }, { doc_count: 1, key: 'completed_with_warnings' }, { doc_count: 1, key: 'failed' }, ], }, + doc_count: 11, + layoutTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'preserve_layout', doc_count: 5 }, { key: 'print', doc_count: 1 }, ] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 3, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 3 }, ] } }, { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, { key: 'csv', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 1 }, ] } }, ] } }, { key: 'completed_with_warnings', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'PNG', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, { key: 'failed', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, ] } }, { key: 'pending', doc_count: 1, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, ] }, + objectTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 6 }, ] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6 }, { key: 'completed_with_warnings', doc_count: 2 }, { key: 'failed', doc_count: 2 }, { key: 'pending', doc_count: 1 }, ] }, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 6, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 1713303.0 }, sizeAvg: { value: 957215.0 }, sizeMin: { value: 43226.0 } }, { key: 'csv_searchsource', doc_count: 3, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 7557.0 }, sizeAvg: { value: 3684.6666666666665 }, sizeMin: { value: 204.0 } }, { key: 'PNG', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 37748.0 }, sizeAvg: { value: 37748.0 }, sizeMin: { value: 37748.0 } }, { key: 'csv', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 231.0 }, sizeAvg: { value: 231.0 }, sizeMin: { value: 231.0 } }, ] }, + sizeMax: { value: 1713303.0 }, + sizeMin: { value: 204.0 }, + sizeAvg: { value: 365084.75 }, }, last7Days: { - doc_count: 1, - jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], }, - statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] }, - }, - lastDay: { - doc_count: 1, - jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], }, - statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] }, + doc_count: 0, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + sizeMax: { value: null }, + sizeMin: { value: null }, + sizeAvg: { value: null }, }, }, - }, + }, // prettier-ignore }, - } as SearchResponse) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); @@ -258,121 +255,21 @@ describe('data modeling', () => { buckets: { all: { doc_count: 9, - layoutTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusByApp: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'completed', - doc_count: 9, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'csv_searchsource', - doc_count: 5, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 5 }], - }, - }, - { - key: 'csv', - doc_count: 4, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 4 }], - }, - }, - ], - }, - }, - ], - }, - objectTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'completed', doc_count: 9 }], - }, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, - { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, - ], - }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] }, + jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] }, }, last7Days: { doc_count: 9, - layoutTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusByApp: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'completed', - doc_count: 9, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'csv_searchsource', - doc_count: 5, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 5 }], - }, - }, - { - key: 'csv', - doc_count: 4, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 4 }], - }, - }, - ], - }, - }, - ], - }, - objectTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'completed', doc_count: 9 }], - }, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, - { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, - ], - }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] }, + jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] }, }, - }, + }, // prettier-ignore }, }, }) @@ -393,39 +290,30 @@ describe('data modeling', () => { } ); collectorFetchContext = getMockFetchClients( - getResponseMock( - { + getResponseMock({ aggregations: { ranges: { buckets: { all: { doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, + layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } }, + statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] }, + objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, + jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] }, }, last7Days: { doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, + layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } }, + statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] }, + objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, + jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] }, }, - lastDay: { - doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, - statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, - }, - }, + }, // prettier-ignore }, }, - } as SearchResponse) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); @@ -445,9 +333,9 @@ describe('data modeling', () => { collectorFetchContext = getMockFetchClients( getResponseMock({ - aggregations: { - ranges: { - buckets: { + aggregations: { + ranges: { + buckets: { all: { doc_count: 0, jobTypes: { buckets: [] }, @@ -455,6 +343,9 @@ describe('data modeling', () => { objectTypes: { doc_count: 0, pdf: { buckets: [] } }, statusByApp: { buckets: [] }, statusTypes: { buckets: [] }, + sizeMax: { value: null}, + sizeMin: { value: null }, + sizeAvg: { value: null}, }, last7Days: { doc_count: 0, @@ -463,19 +354,15 @@ describe('data modeling', () => { objectTypes: { doc_count: 0, pdf: { buckets: [] } }, statusByApp: { buckets: [] }, statusTypes: { buckets: [] }, + sizeMax: { value: null}, + sizeMin: { value: null }, + sizeAvg: { value: null}, + }, - lastDay: { - doc_count: 0, - jobTypes: { buckets: [] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [] }, - statusTypes: { buckets: [] }, - }, + }, // prettier-ignore }, }, - }, - } as SearchResponse) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index 02bf65e7c5e4d..9580ddb935dfb 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -14,6 +14,7 @@ import { LayoutCounts, RangeStats, ReportingUsageType, + SizePercentiles, } from './types'; const appCountsSchema: MakeSchemaFrom = { @@ -39,10 +40,21 @@ const byAppCountsSchema: MakeSchemaFrom = { printable_pdf_v2: appCountsSchema, }; +const sizesSchema: MakeSchemaFrom = { + '1.0': { type: 'long' }, + '5.0': { type: 'long' }, + '25.0': { type: 'long' }, + '50.0': { type: 'long' }, + '75.0': { type: 'long' }, + '95.0': { type: 'long' }, + '99.0': { type: 'long' }, +}; + const availableTotalSchema: MakeSchemaFrom = { available: { type: 'boolean' }, total: { type: 'long' }, deprecated: { type: 'long' }, + sizes: sizesSchema, app: appCountsSchema, layout: layoutCountsSchema, }; @@ -74,6 +86,7 @@ const rangeStatsSchema: MakeSchemaFrom = { pending: byAppCountsSchema, processing: byAppCountsSchema, }, + output_size: sizesSchema, }; export const reportingSchema: MakeSchemaFrom = { diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 7bd79de090b37..856d3ad10cb26 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -5,45 +5,57 @@ * 2.0. */ -export interface KeyCountBucket { - key: string; +export interface SizePercentiles { + '1.0': number | null; + '5.0': number | null; + '25.0': number | null; + '50.0': number | null; + '75.0': number | null; + '95.0': number | null; + '99.0': number | null; +} + +interface DocCount { doc_count: number; - isDeprecated?: { - doc_count: number; - }; +} + +interface SizeStats { + sizes?: { values: SizePercentiles }; +} + +export interface KeyCountBucket extends DocCount, SizeStats { + key: string; + isDeprecated?: DocCount; } export interface AggregationBuckets { buckets: KeyCountBucket[]; } -export interface StatusByAppBucket { +export interface StatusByAppBucket extends DocCount { key: string; - doc_count: number; jobTypes: { - buckets: Array<{ - doc_count: number; - key: string; - appNames: AggregationBuckets; - }>; + buckets: Array< + { + key: string; + appNames: AggregationBuckets; + } & DocCount + >; }; } -export interface AggregationResultBuckets { - jobTypes: AggregationBuckets; +export interface AggregationResultBuckets extends DocCount, SizeStats { + jobTypes?: AggregationBuckets; layoutTypes: { - doc_count: number; - pdf: AggregationBuckets; - }; + pdf?: AggregationBuckets; + } & DocCount; objectTypes: { - doc_count: number; - pdf: AggregationBuckets; - }; + pdf?: AggregationBuckets; + } & DocCount; statusTypes: AggregationBuckets; statusByApp: { buckets: StatusByAppBucket[]; }; - doc_count: number; } export interface SearchResponse { @@ -61,6 +73,7 @@ export interface AvailableTotal { available: boolean; total: number; deprecated?: number; + sizes?: SizePercentiles; app?: { search?: number; dashboard?: number; @@ -110,7 +123,8 @@ type StatusByAppCounts = { export type RangeStats = JobTypes & { _all: number; status: StatusCounts; - statuses: StatusByAppCounts; + statuses?: StatusByAppCounts; + output_size?: SizePercentiles; }; export type ReportingUsageType = RangeStats & { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index ce2c90175a0b7..2dd7b96cfef95 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -4086,6 +4086,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4128,6 +4153,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4170,6 +4220,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4212,6 +4287,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4254,6 +4354,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4296,6 +4421,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4338,6 +4488,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4975,6 +5150,31 @@ } } }, + "output_size": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "available": { "type": "boolean" }, @@ -4997,6 +5197,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5039,6 +5264,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5081,6 +5331,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5123,6 +5398,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5165,6 +5465,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5207,6 +5532,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5249,6 +5599,31 @@ "deprecated": { "type": "long" }, + "sizes": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5885,6 +6260,31 @@ } } } + }, + "output_size": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } } } }