From d3f5f46cd43482c6538f85cb9d39c51d16d535ad Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 17:25:53 -0700 Subject: [PATCH 01/13] [Reporting] Add output size stats to telemetry metrics --- .../server/usage/get_export_stats.ts | 2 ++ .../server/usage/get_reporting_usage.ts | 27 +++++++++++++++++-- .../plugins/reporting/server/usage/types.ts | 10 ++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index ffdb6cdc290d7..20a2a3609229b 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -56,6 +56,7 @@ export const getExportStats = ( _all: rangeAll, status: rangeStatus, statuses: rangeStatusByApp, + output_size: outputSize, ...rangeStats } = rangeStatsInput; @@ -84,6 +85,7 @@ export const getExportStats = ( _all: rangeAll || 0, status: { completed: 0, failed: 0, ...rangeStatus }, statuses: rangeStatusByApp, + output_size: outputSize, } as RangeStats; return resultStats; diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 9aba7841162c2..9d63eb21bc61f 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -33,6 +33,10 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword'; const STATUS_TYPES_KEY = 'statusTypes'; const STATUS_BY_APP_KEY = 'statusByApp'; const STATUS_TYPES_FIELD = 'status'; +const OUTPUT_SIZE_MIN_KEY = 'sizeMin'; +const OUTPUT_SIZE_MAX_KEY = 'sizeMax'; +const OUTPUT_SIZE_AVG_KEY = 'sizeAvg'; +const OUTPUT_SIZE_FIELD = 'output.size'; const DEFAULT_TERMS_SIZE = 10; const PRINTABLE_PDF_JOBTYPE = 'printable_pdf'; @@ -97,7 +101,17 @@ function getAggStats(aggs: AggregationResultBuckets): Partial { statusByApp = getAppStatuses(statusAppBuckets); } - return { _all: all, status: statusTypes, statuses: statusByApp, ...jobTypes }; + return { + _all: all, + status: statusTypes, + statuses: statusByApp, + output_size: { + max: get(aggs[OUTPUT_SIZE_MAX_KEY], 'value'), + min: get(aggs[OUTPUT_SIZE_MIN_KEY], 'value'), + avg: get(aggs[OUTPUT_SIZE_AVG_KEY], 'value'), + }, + ...jobTypes, + }; } type RangeStatSets = Partial & { @@ -161,7 +175,7 @@ export async function getReportingUsage( jobTypes: { terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE }, aggs: { - appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, // NOTE Discover/CSV export is missing the 'meta.objectType' field, so Discover/CSV results are missing for this agg + appNames: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, }, }, }, @@ -174,6 +188,15 @@ export async function getReportingUsage( filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, }, + [OUTPUT_SIZE_MIN_KEY]: { + min: { field: OUTPUT_SIZE_FIELD }, + }, + [OUTPUT_SIZE_MAX_KEY]: { + max: { field: OUTPUT_SIZE_FIELD }, + }, + [OUTPUT_SIZE_AVG_KEY]: { + avg: { field: OUTPUT_SIZE_FIELD }, + }, }, }, }, diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 7bd79de090b37..46610094f31e4 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -44,6 +44,9 @@ export interface AggregationResultBuckets { buckets: StatusByAppBucket[]; }; doc_count: number; + sizeMin: { value: number }; + sizeMax: { value: number }; + sizeAvg: { value: number }; } export interface SearchResponse { @@ -110,7 +113,12 @@ type StatusByAppCounts = { export type RangeStats = JobTypes & { _all: number; status: StatusCounts; - statuses: StatusByAppCounts; + statuses?: StatusByAppCounts; + output_size?: { + max: number; + min: number; + avg: number; + }; }; export type ReportingUsageType = RangeStats & { From 326d1eb8dec7b13e052fc5cd1800fce64028d3ec Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 17:38:11 -0700 Subject: [PATCH 02/13] fix types --- .../usage/reporting_usage_collector.test.ts | 37 +++++++------------ .../plugins/reporting/server/usage/schema.ts | 5 +++ .../plugins/reporting/server/usage/types.ts | 12 +++--- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index e69e56d6272d5..cc6bb0ad8ad23 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -214,6 +214,9 @@ describe('data modeling', () => { objectTypes: { doc_count: 9, pdf: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, }, statusByApp: { buckets: [ { doc_count: 10, jobTypes: { buckets: [ { appNames: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, doc_count: 9, key: 'printable_pdf', }, { appNames: { buckets: [{ doc_count: 1, key: 'visualization' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'failed', }, ], }, statusTypes: { buckets: [ { doc_count: 10, key: 'completed' }, { doc_count: 1, key: 'completed_with_warnings' }, { doc_count: 1, key: 'failed' }, ], }, + sizeMax: { value: 1713835}, + sizeMin: { value: 332 }, + sizeAvg: { value: 356362}, }, last7Days: { doc_count: 1, @@ -222,14 +225,9 @@ describe('data modeling', () => { objectTypes: { doc_count: 0, pdf: { buckets: [] } }, statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], }, statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] }, - }, - lastDay: { - doc_count: 1, - jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], }, - statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] }, + sizeMax: { value: 1738351}, + sizeMin: { value: 332 }, + sizeAvg: { value: 353626}, }, }, }, @@ -414,14 +412,6 @@ describe('data modeling', () => { statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, }, - lastDay: { - doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, - statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, - }, }, }, }, @@ -455,6 +445,9 @@ describe('data modeling', () => { objectTypes: { doc_count: 0, pdf: { buckets: [] } }, statusByApp: { buckets: [] }, statusTypes: { buckets: [] }, + sizeMax: { value: null}, + sizeMin: { value: null }, + sizeAvg: { value: null}, }, last7Days: { doc_count: 0, @@ -463,14 +456,10 @@ describe('data modeling', () => { objectTypes: { doc_count: 0, pdf: { buckets: [] } }, statusByApp: { buckets: [] }, statusTypes: { buckets: [] }, - }, - lastDay: { - doc_count: 0, - jobTypes: { buckets: [] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [] }, - statusTypes: { buckets: [] }, + sizeMax: { value: null}, + sizeMin: { value: null }, + sizeAvg: { value: null}, + }, }, }, diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index 02bf65e7c5e4d..7e87b4de556cc 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -74,6 +74,11 @@ const rangeStatsSchema: MakeSchemaFrom = { pending: byAppCountsSchema, processing: byAppCountsSchema, }, + output_size: { + max: { type: 'long' }, + min: { type: 'long' }, + avg: { type: 'long' }, + }, }; export const reportingSchema: MakeSchemaFrom = { diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 46610094f31e4..4ddfd52008f4a 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -44,9 +44,9 @@ export interface AggregationResultBuckets { buckets: StatusByAppBucket[]; }; doc_count: number; - sizeMin: { value: number }; - sizeMax: { value: number }; - sizeAvg: { value: number }; + sizeMin: { value: number | null }; + sizeMax: { value: number | null }; + sizeAvg: { value: number | null }; } export interface SearchResponse { @@ -115,9 +115,9 @@ export type RangeStats = JobTypes & { status: StatusCounts; statuses?: StatusByAppCounts; output_size?: { - max: number; - min: number; - avg: number; + max: number | null; + min: number | null; + avg: number | null; }; }; From 840ef386ed0c718eb4ea61b630a55bf132549dcb Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 18:28:57 -0700 Subject: [PATCH 03/13] add output_size for each jobtype --- .../server/usage/get_export_stats.ts | 3 ++ .../server/usage/get_reporting_usage.ts | 39 +++++++++++++------ .../plugins/reporting/server/usage/types.ts | 18 ++++++--- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index 20a2a3609229b..df700e5913235 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -13,6 +13,7 @@ const jobTypeIsDeprecated = (jobType: string) => DEPRECATED_JOB_TYPES.includes(j const defaultTotalsForFeature: Omit = { total: 0, deprecated: 0, + output_size: { min: null, max: null, avg: null }, app: { 'canvas workpad': 0, search: 0, visualization: 0, dashboard: 0 }, layout: { canvas: 0, print: 0, preserve_layout: 0 }, }; @@ -27,12 +28,14 @@ function getAvailableTotalForFeature( ): AvailableTotal { // if the type itself is deprecated, all jobs are deprecated, otherwise only some of them might be const deprecated = jobTypeIsDeprecated(typeKey) ? jobType.total : jobType.deprecated || 0; + const { max: sizeMax, min: sizeMin, avg: sizeAvg } = jobType.output_size; // merge the additional stats for the jobType const availableTotal = { available: isAvailable(featureAvailability, typeKey), total: jobType.total, deprecated, + output_size: { min: sizeMin, max: sizeMax, avg: sizeAvg }, app: { ...defaultTotalsForFeature.app, ...jobType.app }, layout: { ...defaultTotalsForFeature.layout, ...jobType.layout }, }; diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 9d63eb21bc61f..7454cee28a179 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -70,11 +70,16 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) => function getAggStats(aggs: AggregationResultBuckets): Partial { const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY]; const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => { - const { key, doc_count: count, isDeprecated } = bucket; + const { key, doc_count: count, isDeprecated, sizeMax, sizeMin, sizeAvg } = bucket; const deprecatedCount = isDeprecated?.doc_count; const total: Omit = { total: count, deprecated: deprecatedCount, + output_size: { + max: sizeMax.value, + min: sizeMin.value, + avg: sizeAvg.value, + }, }; return { ...accum, [key]: total }; }, {} as JobTypes); @@ -150,6 +155,18 @@ export async function getReportingUsage( ): Promise { const reportingIndex = config.get('index'); + const outputSizeAggs = { + [OUTPUT_SIZE_MIN_KEY]: { + min: { field: OUTPUT_SIZE_FIELD }, + }, + [OUTPUT_SIZE_MAX_KEY]: { + max: { field: OUTPUT_SIZE_FIELD }, + }, + [OUTPUT_SIZE_AVG_KEY]: { + avg: { field: OUTPUT_SIZE_FIELD }, + }, + }; + const params = { index: `${reportingIndex}-*`, filterPath: 'aggregations.*.buckets', @@ -166,8 +183,12 @@ export async function getReportingUsage( aggs: { [JOB_TYPES_KEY]: { terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE }, - aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } } }, + aggs: { + isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } }, + ...outputSizeAggs, + }, }, + [STATUS_TYPES_KEY]: { terms: { field: STATUS_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, [STATUS_BY_APP_KEY]: { terms: { field: 'status', size: DEFAULT_TERMS_SIZE }, @@ -182,21 +203,15 @@ export async function getReportingUsage( }, [OBJECT_TYPES_KEY]: { filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, - aggs: { pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, + aggs: { + pdf: { terms: { field: OBJECT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } }, + }, }, [LAYOUT_TYPES_KEY]: { filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, }, - [OUTPUT_SIZE_MIN_KEY]: { - min: { field: OUTPUT_SIZE_FIELD }, - }, - [OUTPUT_SIZE_MAX_KEY]: { - max: { field: OUTPUT_SIZE_FIELD }, - }, - [OUTPUT_SIZE_AVG_KEY]: { - avg: { field: OUTPUT_SIZE_FIELD }, - }, + ...outputSizeAggs, }, }, }, diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 4ddfd52008f4a..7af3d636b7b4e 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -5,7 +5,13 @@ * 2.0. */ -export interface KeyCountBucket { +interface SizeStats { + sizeMin: { value: number | null }; + sizeMax: { value: number | null }; + sizeAvg: { value: number | null }; +} + +export interface KeyCountBucket extends SizeStats { key: string; doc_count: number; isDeprecated?: { @@ -29,7 +35,7 @@ export interface StatusByAppBucket { }; } -export interface AggregationResultBuckets { +export interface AggregationResultBuckets extends SizeStats { jobTypes: AggregationBuckets; layoutTypes: { doc_count: number; @@ -44,9 +50,6 @@ export interface AggregationResultBuckets { buckets: StatusByAppBucket[]; }; doc_count: number; - sizeMin: { value: number | null }; - sizeMax: { value: number | null }; - sizeAvg: { value: number | null }; } export interface SearchResponse { @@ -64,6 +67,11 @@ export interface AvailableTotal { available: boolean; total: number; deprecated?: number; + output_size: { + max: number | null; + min: number | null; + avg: number | null; + }; app?: { search?: number; dashboard?: number; From ad7a62cf494fc87a91ef8c78f5f9523a1a671eea Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 18:52:13 -0700 Subject: [PATCH 04/13] add size metrics for each job type --- .../server/usage/get_export_stats.test.ts | 27 ++++++++++++++++--- .../plugins/reporting/server/usage/schema.ts | 14 ++++++---- .../plugins/reporting/server/usage/types.ts | 24 ++++++++--------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index 782f2e910038e..d5570c08db686 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -67,14 +67,19 @@ test('Model of job status and status-by-pdf-app', () => { test('Model of jobTypes', () => { const result = getExportStats( { - PNG: { available: true, total: 3 }, + PNG: { available: true, total: 3, output_size: { max: 999, min: 333, avg: 666 } }, printable_pdf: { available: true, total: 3, + output_size: { max: 999, min: 333, avg: 666 }, app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 }, layout: { preserve_layout: 3, print: 0 }, }, - csv_searchsource: { available: true, total: 3 }, + csv_searchsource: { + available: true, + total: 3, + output_size: { max: 999, min: 333, avg: 666 }, + }, }, featureMap, exportTypesHandler @@ -156,7 +161,14 @@ test('Model of jobTypes', () => { test('PNG counts, provided count of deprecated jobs explicitly', () => { const result = getExportStats( - { PNG: { available: true, total: 15, deprecated: 5 } }, + { + PNG: { + available: true, + total: 15, + deprecated: 5, + output_size: { min: 111, max: 333, avg: 222 }, + }, + }, featureMap, exportTypesHandler ); @@ -182,7 +194,14 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => { const result = getExportStats( - { csv: { available: true, total: 15, deprecated: 0 } }, + { + csv: { + available: true, + total: 15, + deprecated: 0, + output_size: { min: 111, max: 333, avg: 222 }, + }, + }, featureMap, exportTypesHandler ); diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index 7e87b4de556cc..be22071e2f7f7 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -14,6 +14,7 @@ import { LayoutCounts, RangeStats, ReportingUsageType, + SizeMetrics, } from './types'; const appCountsSchema: MakeSchemaFrom = { @@ -39,10 +40,17 @@ const byAppCountsSchema: MakeSchemaFrom = { printable_pdf_v2: appCountsSchema, }; +const sizesSchema: MakeSchemaFrom = { + max: { type: 'long' }, + min: { type: 'long' }, + avg: { type: 'long' }, +}; + const availableTotalSchema: MakeSchemaFrom = { available: { type: 'boolean' }, total: { type: 'long' }, deprecated: { type: 'long' }, + output_size: sizesSchema, app: appCountsSchema, layout: layoutCountsSchema, }; @@ -74,11 +82,7 @@ const rangeStatsSchema: MakeSchemaFrom = { pending: byAppCountsSchema, processing: byAppCountsSchema, }, - output_size: { - max: { type: 'long' }, - min: { type: 'long' }, - avg: { type: 'long' }, - }, + output_size: sizesSchema, }; export const reportingSchema: MakeSchemaFrom = { diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 7af3d636b7b4e..a5b958eeb0951 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -6,9 +6,9 @@ */ interface SizeStats { - sizeMin: { value: number | null }; - sizeMax: { value: number | null }; - sizeAvg: { value: number | null }; + sizeMin?: { value: number | null }; + sizeMax?: { value: number | null }; + sizeAvg?: { value: number | null }; } export interface KeyCountBucket extends SizeStats { @@ -63,15 +63,17 @@ export interface SearchResponse { }; } +export interface SizeMetrics { + max: number | null; + min: number | null; + avg: number | null; +} + export interface AvailableTotal { available: boolean; total: number; deprecated?: number; - output_size: { - max: number | null; - min: number | null; - avg: number | null; - }; + output_size: SizeMetrics; app?: { search?: number; dashboard?: number; @@ -122,11 +124,7 @@ export type RangeStats = JobTypes & { _all: number; status: StatusCounts; statuses?: StatusByAppCounts; - output_size?: { - max: number | null; - min: number | null; - avg: number | null; - }; + output_size: SizeMetrics; }; export type ReportingUsageType = RangeStats & { From 0819f15f210294678daa919ce7227b96b01ee79f Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 18:58:50 -0700 Subject: [PATCH 05/13] use more mock data in unit tests --- .../server/usage/get_reporting_usage.ts | 15 +++---- .../usage/reporting_usage_collector.test.ts | 42 +++++++++---------- .../plugins/reporting/server/usage/types.ts | 6 +-- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 7454cee28a179..25ac4516c2780 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -13,6 +13,7 @@ import type { GetLicense } from './'; import { getExportStats } from './get_export_stats'; import { getExportTypesHandler } from './get_export_type_handler'; import type { + AggregationBuckets, AggregationResultBuckets, AvailableTotal, FeatureAvailabilityMap, @@ -68,7 +69,7 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) => }, {}); function getAggStats(aggs: AggregationResultBuckets): Partial { - const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY]; + const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets; const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => { const { key, doc_count: count, isDeprecated, sizeMax, sizeMin, sizeAvg } = bucket; const deprecatedCount = isDeprecated?.doc_count; @@ -76,9 +77,9 @@ function getAggStats(aggs: AggregationResultBuckets): Partial { total: count, deprecated: deprecatedCount, output_size: { - max: sizeMax.value, - min: sizeMin.value, - avg: sizeAvg.value, + max: sizeMax?.value ?? null, + min: sizeMin?.value ?? null, + avg: sizeAvg?.value ?? null, }, }; return { ...accum, [key]: total }; @@ -111,9 +112,9 @@ function getAggStats(aggs: AggregationResultBuckets): Partial { status: statusTypes, statuses: statusByApp, output_size: { - max: get(aggs[OUTPUT_SIZE_MAX_KEY], 'value'), - min: get(aggs[OUTPUT_SIZE_MIN_KEY], 'value'), - avg: get(aggs[OUTPUT_SIZE_AVG_KEY], 'value'), + max: get(aggs[OUTPUT_SIZE_MAX_KEY], 'value') ?? null, + min: get(aggs[OUTPUT_SIZE_MIN_KEY], 'value') ?? null, + avg: get(aggs[OUTPUT_SIZE_AVG_KEY], 'value') ?? null, }, ...jobTypes, }; diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index cc6bb0ad8ad23..a467a9350cb77 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -202,37 +202,37 @@ describe('data modeling', () => { } ); collectorFetchContext = getMockFetchClients( - getResponseMock( - { + getResponseMock({ aggregations: { ranges: { + meta: {}, buckets: { all: { - doc_count: 12, - jobTypes: { buckets: [ { doc_count: 9, key: 'printable_pdf' }, { doc_count: 3, key: 'PNG' }, ], }, - layoutTypes: { doc_count: 9, pdf: { buckets: [{ doc_count: 9, key: 'preserve_layout' }] }, }, - objectTypes: { doc_count: 9, pdf: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, }, - statusByApp: { buckets: [ { doc_count: 10, jobTypes: { buckets: [ { appNames: { buckets: [ { doc_count: 6, key: 'canvas workpad' }, { doc_count: 3, key: 'visualization' }, ], }, doc_count: 9, key: 'printable_pdf', }, { appNames: { buckets: [{ doc_count: 1, key: 'visualization' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'failed', }, ], }, - statusTypes: { buckets: [ { doc_count: 10, key: 'completed' }, { doc_count: 1, key: 'completed_with_warnings' }, { doc_count: 1, key: 'failed' }, ], }, - sizeMax: { value: 1713835}, - sizeMin: { value: 332 }, - sizeAvg: { value: 356362}, + doc_count: 11, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 6, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 1713303.0 }, sizeAvg: { value: 957215.0 }, sizeMin: { value: 43226.0 }, }, { key: 'csv_searchsource', doc_count: 3, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 7557.0 }, sizeAvg: { value: 3684.6666666666665 }, sizeMin: { value: 204.0 }, }, { key: 'PNG', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 37748.0 }, sizeAvg: { value: 37748.0 }, sizeMin: { value: 37748.0 }, }, { key: 'csv', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 231.0 }, sizeAvg: { value: 231.0 }, sizeMin: { value: 231.0 }, }, ], }, + layoutTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'preserve_layout', doc_count: 5 }, { key: 'print', doc_count: 1 }, ], }, }, + objectTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 6 }], }, }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 3, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 3 }], }, }, { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 2 }], }, }, { key: 'csv', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 1 }], }, }, ], }, }, { key: 'completed_with_warnings', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'PNG', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, ], }, }, { key: 'failed', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 2 }], }, }, ], }, }, { key: 'pending', doc_count: 1, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, ], }, }, ], }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6 }, { key: 'completed_with_warnings', doc_count: 2 }, { key: 'failed', doc_count: 2 }, { key: 'pending', doc_count: 1 }, ], }, + sizeMax: { value: 1713303.0 }, + sizeAvg: { value: 365084.75 }, + sizeMin: { value: 204.0 }, }, last7Days: { - doc_count: 1, - jobTypes: { buckets: [{ doc_count: 1, key: 'PNG' }] }, - layoutTypes: { doc_count: 0, pdf: { buckets: [] } }, - objectTypes: { doc_count: 0, pdf: { buckets: [] } }, - statusByApp: { buckets: [ { doc_count: 1, jobTypes: { buckets: [ { appNames: { buckets: [{ doc_count: 1, key: 'dashboard' }] }, doc_count: 1, key: 'PNG', }, ], }, key: 'completed_with_warnings', }, ], }, - statusTypes: { buckets: [{ doc_count: 1, key: 'completed_with_warnings' }] }, - sizeMax: { value: 1738351}, - sizeMin: { value: 332 }, - sizeAvg: { value: 353626}, + doc_count: 0, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, + sizeMax: { value: null }, + sizeAvg: { value: null }, + sizeMin: { value: null }, }, }, }, }, - } as SearchResponse) // prettier-ignore + }) // prettier-ignore ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index a5b958eeb0951..a40e187abdd72 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -36,14 +36,14 @@ export interface StatusByAppBucket { } export interface AggregationResultBuckets extends SizeStats { - jobTypes: AggregationBuckets; + jobTypes?: AggregationBuckets; layoutTypes: { doc_count: number; - pdf: AggregationBuckets; + pdf?: AggregationBuckets; }; objectTypes: { doc_count: number; - pdf: AggregationBuckets; + pdf?: AggregationBuckets; }; statusTypes: AggregationBuckets; statusByApp: { From c29c96b54956acaf1c7986bbf75a72e269ac0113 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 19:10:08 -0700 Subject: [PATCH 06/13] clean up test --- .../usage/reporting_usage_collector.test.ts | 186 ++++-------------- 1 file changed, 42 insertions(+), 144 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index a467a9350cb77..447085810cfd0 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -18,7 +18,6 @@ import { getReportingUsageCollector, registerReportingUsageCollector, } from './reporting_usage_collector'; -import { SearchResponse } from './types'; const exportTypesRegistry = getExportTypesRegistry(); @@ -190,7 +189,7 @@ describe('data modeling', () => { beforeAll(async () => { mockCore = await createMockReportingCore(createMockConfigSchema()); }); - test('with normal looking usage data', async () => { + test('with usage data from the reporting/archived_reports es archive', async () => { const plugins = getPluginsMock(); const collector = getReportingUsageCollector( mockCore, @@ -209,30 +208,30 @@ describe('data modeling', () => { buckets: { all: { doc_count: 11, - jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 6, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 1713303.0 }, sizeAvg: { value: 957215.0 }, sizeMin: { value: 43226.0 }, }, { key: 'csv_searchsource', doc_count: 3, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 7557.0 }, sizeAvg: { value: 3684.6666666666665 }, sizeMin: { value: 204.0 }, }, { key: 'PNG', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 37748.0 }, sizeAvg: { value: 37748.0 }, sizeMin: { value: 37748.0 }, }, { key: 'csv', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 231.0 }, sizeAvg: { value: 231.0 }, sizeMin: { value: 231.0 }, }, ], }, - layoutTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'preserve_layout', doc_count: 5 }, { key: 'print', doc_count: 1 }, ], }, }, - objectTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 6 }], }, }, - statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 3, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 3 }], }, }, { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 2 }], }, }, { key: 'csv', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 1 }], }, }, ], }, }, { key: 'completed_with_warnings', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'PNG', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, ], }, }, { key: 'failed', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 2 }], }, }, ], }, }, { key: 'pending', doc_count: 1, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'dashboard', doc_count: 1 }], }, }, ], }, }, ], }, - statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6 }, { key: 'completed_with_warnings', doc_count: 2 }, { key: 'failed', doc_count: 2 }, { key: 'pending', doc_count: 1 }, ], }, + layoutTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'preserve_layout', doc_count: 5 }, { key: 'print', doc_count: 1 }, ] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 3, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 3 }, ] } }, { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, { key: 'csv', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'search', doc_count: 1 }, ] } }, ] } }, { key: 'completed_with_warnings', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'PNG', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, { key: 'failed', doc_count: 2, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 2 }, ] } }, ] } }, { key: 'pending', doc_count: 1, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 1, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 1 }, ] } }, ] } }, ] }, + objectTypes: { doc_count: 6, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'dashboard', doc_count: 6 }, ] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 6 }, { key: 'completed_with_warnings', doc_count: 2 }, { key: 'failed', doc_count: 2 }, { key: 'pending', doc_count: 1 }, ] }, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'printable_pdf', doc_count: 6, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 1713303.0 }, sizeAvg: { value: 957215.0 }, sizeMin: { value: 43226.0 } }, { key: 'csv_searchsource', doc_count: 3, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 7557.0 }, sizeAvg: { value: 3684.6666666666665 }, sizeMin: { value: 204.0 } }, { key: 'PNG', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 37748.0 }, sizeAvg: { value: 37748.0 }, sizeMin: { value: 37748.0 } }, { key: 'csv', doc_count: 1, isDeprecated: { meta: {}, doc_count: 0 }, sizeMax: { value: 231.0 }, sizeAvg: { value: 231.0 }, sizeMin: { value: 231.0 } }, ] }, sizeMax: { value: 1713303.0 }, - sizeAvg: { value: 365084.75 }, sizeMin: { value: 204.0 }, + sizeAvg: { value: 365084.75 }, }, last7Days: { doc_count: 0, - jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, - layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, }, - objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, }, - statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, - statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + jobTypes: { meta: {}, doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, sizeMax: { value: null }, - sizeAvg: { value: null }, sizeMin: { value: null }, + sizeAvg: { value: null }, }, }, - }, + }, // prettier-ignore }, - }) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); @@ -256,121 +255,21 @@ describe('data modeling', () => { buckets: { all: { doc_count: 9, - layoutTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusByApp: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'completed', - doc_count: 9, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'csv_searchsource', - doc_count: 5, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 5 }], - }, - }, - { - key: 'csv', - doc_count: 4, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 4 }], - }, - }, - ], - }, - }, - ], - }, - objectTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'completed', doc_count: 9 }], - }, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, - { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, - ], - }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] }, + jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] }, }, last7Days: { doc_count: 9, - layoutTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusByApp: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'completed', - doc_count: 9, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: 'csv_searchsource', - doc_count: 5, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 5 }], - }, - }, - { - key: 'csv', - doc_count: 4, - appNames: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'search', doc_count: 4 }], - }, - }, - ], - }, - }, - ], - }, - objectTypes: { - doc_count: 0, - pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - statusTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [{ key: 'completed', doc_count: 9 }], - }, - jobTypes: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, - { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, - ], - }, + layoutTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusByApp: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'completed', doc_count: 9, jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 5 }] } }, { key: 'csv', doc_count: 4, appNames: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'search', doc_count: 4 }] } }, ] } }, ] }, + objectTypes: { doc_count: 0, pdf: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] } }, + statusTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [{ key: 'completed', doc_count: 9 }] }, + jobTypes: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [ { key: 'csv_searchsource', doc_count: 5, isDeprecated: { doc_count: 0 } }, { key: 'csv', doc_count: 4, isDeprecated: { doc_count: 4 } }, ] }, }, - }, + }, // prettier-ignore }, }, }) @@ -391,31 +290,30 @@ describe('data modeling', () => { } ); collectorFetchContext = getMockFetchClients( - getResponseMock( - { + getResponseMock({ aggregations: { ranges: { buckets: { all: { doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, + layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } }, + statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] }, + objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, + jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] }, }, last7Days: { doc_count: 4, - layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] }, }, - statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] }, }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ], }, }, ], }, - objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ], }, }, + layoutTypes: { doc_count: 2, pdf: { buckets: [{ key: 'preserve_layout', doc_count: 2 }] } }, + statusByApp: { buckets: [ { key: 'completed', doc_count: 4, jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2, appNames: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, { key: 'PNG', doc_count: 1, appNames: { buckets: [{ key: 'dashboard', doc_count: 1 }] } }, { key: 'csv', doc_count: 1, appNames: { buckets: [] } }, ] } }, ] }, + objectTypes: { doc_count: 2, pdf: { buckets: [ { key: 'canvas workpad', doc_count: 1 }, { key: 'dashboard', doc_count: 1 }, ] } }, statusTypes: { buckets: [{ key: 'completed', doc_count: 4 }] }, - jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ], }, + jobTypes: { buckets: [ { key: 'printable_pdf', doc_count: 2 }, { key: 'PNG', doc_count: 1 }, { key: 'csv', doc_count: 1 }, ] }, }, - }, + }, // prettier-ignore }, }, - } as SearchResponse) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); @@ -435,9 +333,9 @@ describe('data modeling', () => { collectorFetchContext = getMockFetchClients( getResponseMock({ - aggregations: { - ranges: { - buckets: { + aggregations: { + ranges: { + buckets: { all: { doc_count: 0, jobTypes: { buckets: [] }, @@ -461,10 +359,10 @@ describe('data modeling', () => { sizeAvg: { value: null}, }, + }, // prettier-ignore }, }, - }, - } as SearchResponse) // prettier-ignore + }) ); const usageStats = await collector.fetch(collectorFetchContext); expect(usageStats).toMatchSnapshot(); From a2179e6aab6ab02996fcde72f0879c9258123ef7 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 19:19:51 -0700 Subject: [PATCH 07/13] update test snapshots --- .../reporting_usage_collector.test.ts.snap | 636 ++++++++++++++++-- .../server/usage/get_export_stats.test.ts | 30 + 2 files changed, 598 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 12debe5c85d5e..96b352b9977f1 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -37,6 +37,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -73,6 +84,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -118,6 +140,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -154,6 +187,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -190,6 +234,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -230,6 +285,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -266,6 +332,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -305,6 +382,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -341,6 +429,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -377,10 +476,32 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "printable_pdf": Object { "app": Object { "canvas workpad": Object { @@ -413,6 +534,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -449,6 +581,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -973,6 +1116,17 @@ Object { }, }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "printable_pdf": Object { "app": Object { "canvas workpad": Object { @@ -1005,6 +1159,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -1041,6 +1206,17 @@ Object { "type": "long", }, }, + "output_size": Object { + "avg": Object { + "type": "long", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, "total": Object { "type": "long", }, @@ -1585,6 +1761,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "PNGV2": Object { @@ -1601,6 +1782,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "_all": 9, @@ -1620,6 +1806,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 4, }, "csv_searchsource": Object { @@ -1636,6 +1827,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1652,6 +1848,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "enabled": true, @@ -1670,6 +1871,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "PNGV2": Object { @@ -1686,6 +1892,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "_all": 9, @@ -1703,6 +1914,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 4, }, "csv_searchsource": Object { @@ -1719,6 +1935,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1735,8 +1956,18 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -1751,6 +1982,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "printable_pdf_v2": Object { @@ -1767,6 +2003,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { @@ -1784,6 +2025,11 @@ Object { }, }, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -1798,6 +2044,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "printable_pdf_v2": Object { @@ -1814,6 +2065,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { @@ -1849,6 +2105,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "PNGV2": Object { @@ -1865,6 +2126,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "_all": 0, @@ -1884,6 +2150,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource": Object { @@ -1900,6 +2171,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource_immediate": Object { @@ -1916,6 +2192,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "enabled": true, @@ -1934,6 +2215,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "PNGV2": Object { @@ -1950,6 +2236,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "_all": 0, @@ -1967,6 +2258,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource": Object { @@ -1983,6 +2279,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource_immediate": Object { @@ -1999,8 +2300,18 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2015,6 +2326,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "printable_pdf_v2": Object { @@ -2031,6 +2347,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { @@ -2039,6 +2360,11 @@ Object { }, "statuses": Object {}, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2053,6 +2379,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "printable_pdf_v2": Object { @@ -2069,6 +2400,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { @@ -2079,7 +2415,7 @@ Object { } `; -exports[`data modeling with normal looking usage data 1`] = ` +exports[`data modeling with sparse data 1`] = ` Object { "PNG": Object { "app": Object { @@ -2095,7 +2431,12 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 3, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 1, }, "PNGV2": Object { "app": Object { @@ -2111,9 +2452,14 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, - "_all": 12, + "_all": 4, "available": true, "browser_type": undefined, "csv": Object { @@ -2124,13 +2470,18 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 0, + "deprecated": 1, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 1, }, "csv_searchsource": Object { "app": Object { @@ -2146,6 +2497,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2162,6 +2518,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "enabled": true, @@ -2180,6 +2541,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 1, }, "PNGV2": Object { @@ -2196,9 +2562,14 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, - "_all": 1, + "_all": 4, "csv": Object { "app": Object { "canvas workpad": 0, @@ -2207,13 +2578,18 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 0, + "deprecated": 1, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 1, }, "csv_searchsource": Object { "app": Object { @@ -2229,6 +2605,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2245,12 +2626,22 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { - "canvas workpad": 0, - "dashboard": 0, + "canvas workpad": 1, + "dashboard": 1, "search": 0, "visualization": 0, }, @@ -2258,10 +2649,15 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 0, + "preserve_layout": 2, "print": 0, }, - "total": 0, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 2, }, "printable_pdf_v2": Object { "app": Object { @@ -2277,36 +2673,55 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { - "completed": 0, - "completed_with_warnings": 1, + "completed": 4, "failed": 0, }, "statuses": Object { - "completed_with_warnings": Object { + "completed": Object { "PNG": Object { "dashboard": 1, }, + "csv": Object {}, + "printable_pdf": Object { + "canvas workpad": 1, + "dashboard": 1, + }, }, }, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { - "canvas workpad": 6, - "dashboard": 0, + "canvas workpad": 1, + "dashboard": 1, "search": 0, - "visualization": 3, + "visualization": 0, }, "available": true, "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 9, + "preserve_layout": 2, "print": 0, }, - "total": 9, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 2, }, "printable_pdf_v2": Object { "app": Object { @@ -2322,30 +2737,25 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { - "completed": 10, - "completed_with_warnings": 1, - "failed": 1, + "completed": 4, + "failed": 0, }, "statuses": Object { "completed": Object { - "PNG": Object { - "visualization": 1, - }, - "printable_pdf": Object { - "canvas workpad": 6, - "visualization": 3, - }, - }, - "completed_with_warnings": Object { "PNG": Object { "dashboard": 1, }, - }, - "failed": Object { - "PNG": Object { + "csv": Object {}, + "printable_pdf": Object { + "canvas workpad": 1, "dashboard": 1, }, }, @@ -2353,7 +2763,7 @@ Object { } `; -exports[`data modeling with sparse data 1`] = ` +exports[`data modeling with usage data from the reporting/archived_reports es archive 1`] = ` Object { "PNG": Object { "app": Object { @@ -2369,6 +2779,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 37748, + "max": 37748, + "min": 37748, + }, "total": 1, }, "PNGV2": Object { @@ -2385,9 +2800,14 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, - "_all": 4, + "_all": 11, "available": true, "browser_type": undefined, "csv": Object { @@ -2404,6 +2824,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 231, + "max": 231, + "min": 231, + }, "total": 1, }, "csv_searchsource": Object { @@ -2420,7 +2845,12 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 0, + "output_size": Object { + "avg": 3684.6666666666665, + "max": 7557, + "min": 204, + }, + "total": 3, }, "csv_searchsource_immediate": Object { "app": Object { @@ -2436,6 +2866,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "enabled": true, @@ -2454,7 +2889,12 @@ Object { "preserve_layout": 0, "print": 0, }, - "total": 1, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 0, }, "PNGV2": Object { "app": Object { @@ -2470,9 +2910,14 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, - "_all": 4, + "_all": 0, "csv": Object { "app": Object { "canvas workpad": 0, @@ -2481,13 +2926,18 @@ Object { "visualization": 0, }, "available": true, - "deprecated": 1, + "deprecated": 0, "layout": Object { "canvas": 0, "preserve_layout": 0, "print": 0, }, - "total": 1, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 0, }, "csv_searchsource": Object { "app": Object { @@ -2503,6 +2953,11 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2519,12 +2974,22 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "printable_pdf": Object { "app": Object { - "canvas workpad": 1, - "dashboard": 1, + "canvas workpad": 0, + "dashboard": 0, "search": 0, "visualization": 0, }, @@ -2532,10 +2997,15 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 2, + "preserve_layout": 0, "print": 0, }, - "total": 2, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, + "total": 0, }, "printable_pdf_v2": Object { "app": Object { @@ -2551,29 +3021,28 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { - "completed": 4, + "completed": 0, "failed": 0, }, - "statuses": Object { - "completed": Object { - "PNG": Object { - "dashboard": 1, - }, - "csv": Object {}, - "printable_pdf": Object { - "canvas workpad": 1, - "dashboard": 1, - }, - }, - }, + "statuses": Object {}, + }, + "output_size": Object { + "avg": 365084.75, + "max": 1713303, + "min": 204, }, "printable_pdf": Object { "app": Object { - "canvas workpad": 1, - "dashboard": 1, + "canvas workpad": 0, + "dashboard": 6, "search": 0, "visualization": 0, }, @@ -2581,10 +3050,15 @@ Object { "deprecated": 0, "layout": Object { "canvas": 0, - "preserve_layout": 2, - "print": 0, + "preserve_layout": 5, + "print": 1, }, - "total": 2, + "output_size": Object { + "avg": 957215, + "max": 1713303, + "min": 43226, + }, + "total": 6, }, "printable_pdf_v2": Object { "app": Object { @@ -2600,20 +3074,46 @@ Object { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, }, "status": Object { - "completed": 4, - "failed": 0, + "completed": 6, + "completed_with_warnings": 2, + "failed": 2, + "pending": 1, }, "statuses": Object { "completed": Object { + "csv": Object { + "search": 1, + }, + "csv_searchsource": Object { + "search": 3, + }, + "printable_pdf": Object { + "dashboard": 2, + }, + }, + "completed_with_warnings": Object { "PNG": Object { "dashboard": 1, }, - "csv": Object {}, "printable_pdf": Object { - "canvas workpad": 1, + "dashboard": 1, + }, + }, + "failed": Object { + "printable_pdf": Object { + "dashboard": 2, + }, + }, + "pending": Object { + "printable_pdf": Object { "dashboard": 1, }, }, diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index d5570c08db686..e3f6d37bc50b7 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -100,6 +100,11 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 666, + "max": 999, + "min": 333, + }, "total": 3, } `); @@ -118,6 +123,11 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": null, + "max": null, + "min": null, + }, "total": 0, } `); @@ -136,6 +146,11 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 666, + "max": 999, + "min": 333, + }, "total": 3, } `); @@ -154,6 +169,11 @@ test('Model of jobTypes', () => { "preserve_layout": 3, "print": 0, }, + "output_size": Object { + "avg": 666, + "max": 999, + "min": 333, + }, "total": 3, } `); @@ -187,6 +207,11 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 222, + "max": 333, + "min": 111, + }, "total": 15, } `); @@ -220,6 +245,11 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => "preserve_layout": 0, "print": 0, }, + "output_size": Object { + "avg": 222, + "max": 333, + "min": 111, + }, "total": 15, } `); From c5f3fe62dc97541e963d0c975c77df403c6efa6d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 Sep 2021 20:51:47 -0700 Subject: [PATCH 08/13] update telemetry mapping --- .../schema/xpack_plugins.json | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 0f9bc43cdf37d..a7298008c1ba9 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -4051,6 +4051,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4093,6 +4106,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4135,6 +4161,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4177,6 +4216,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4219,6 +4271,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4261,6 +4326,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4303,6 +4381,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -4940,6 +5031,19 @@ } } }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "available": { "type": "boolean" }, @@ -4962,6 +5066,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5004,6 +5121,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5046,6 +5176,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5088,6 +5231,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5130,6 +5286,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5172,6 +5341,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5214,6 +5396,19 @@ "deprecated": { "type": "long" }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } + }, "app": { "properties": { "search": { @@ -5850,6 +6045,19 @@ } } } + }, + "output_size": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + }, + "avg": { + "type": "long" + } + } } } } From 3d6ecd06c495170a4cd0273de854c64a98e7a77d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 22 Sep 2021 13:13:27 -0700 Subject: [PATCH 09/13] SizeMetrics => SizePercentiles --- .../server/usage/get_export_stats.test.ts | 10 +- .../server/usage/get_export_stats.ts | 5 +- .../server/usage/get_reporting_usage.ts | 41 +-- .../plugins/reporting/server/usage/schema.ts | 16 +- .../plugins/reporting/server/usage/types.ts | 24 +- .../schema/xpack_plugins.json | 316 ++++++++++++++---- 6 files changed, 295 insertions(+), 117 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index e3f6d37bc50b7..396d792f7cc5e 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -67,18 +67,18 @@ test('Model of job status and status-by-pdf-app', () => { test('Model of jobTypes', () => { const result = getExportStats( { - PNG: { available: true, total: 3, output_size: { max: 999, min: 333, avg: 666 } }, + PNG: { available: true, total: 3, sizes: { max: 999, min: 333, avg: 666 } }, printable_pdf: { available: true, total: 3, - output_size: { max: 999, min: 333, avg: 666 }, + sizes: { max: 999, min: 333, avg: 666 }, app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 }, layout: { preserve_layout: 3, print: 0 }, }, csv_searchsource: { available: true, total: 3, - output_size: { max: 999, min: 333, avg: 666 }, + sizes: { max: 999, min: 333, avg: 666 }, }, }, featureMap, @@ -186,7 +186,7 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { available: true, total: 15, deprecated: 5, - output_size: { min: 111, max: 333, avg: 222 }, + sizes: { min: 111, max: 333, avg: 222 }, }, }, featureMap, @@ -224,7 +224,7 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => available: true, total: 15, deprecated: 0, - output_size: { min: 111, max: 333, avg: 222 }, + sizes: { min: 111, max: 333, avg: 222 }, }, }, featureMap, diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index df700e5913235..34ba25fc4ce20 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -13,7 +13,7 @@ const jobTypeIsDeprecated = (jobType: string) => DEPRECATED_JOB_TYPES.includes(j const defaultTotalsForFeature: Omit = { total: 0, deprecated: 0, - output_size: { min: null, max: null, avg: null }, + sizes: undefined, app: { 'canvas workpad': 0, search: 0, visualization: 0, dashboard: 0 }, layout: { canvas: 0, print: 0, preserve_layout: 0 }, }; @@ -28,14 +28,13 @@ function getAvailableTotalForFeature( ): AvailableTotal { // if the type itself is deprecated, all jobs are deprecated, otherwise only some of them might be const deprecated = jobTypeIsDeprecated(typeKey) ? jobType.total : jobType.deprecated || 0; - const { max: sizeMax, min: sizeMin, avg: sizeAvg } = jobType.output_size; // merge the additional stats for the jobType const availableTotal = { available: isAvailable(featureAvailability, typeKey), total: jobType.total, deprecated, - output_size: { min: sizeMin, max: sizeMax, avg: sizeAvg }, + output_size: jobType.sizes, app: { ...defaultTotalsForFeature.app, ...jobType.app }, layout: { ...defaultTotalsForFeature.layout, ...jobType.layout }, }; diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 25ac4516c2780..9a452943ff699 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -34,10 +34,8 @@ const OBJECT_TYPES_FIELD = 'meta.objectType.keyword'; const STATUS_TYPES_KEY = 'statusTypes'; const STATUS_BY_APP_KEY = 'statusByApp'; const STATUS_TYPES_FIELD = 'status'; -const OUTPUT_SIZE_MIN_KEY = 'sizeMin'; -const OUTPUT_SIZE_MAX_KEY = 'sizeMax'; -const OUTPUT_SIZE_AVG_KEY = 'sizeAvg'; -const OUTPUT_SIZE_FIELD = 'output.size'; +const OUTPUT_SIZES_KEY = 'sizes'; +const OUTPUT_SIZES_FIELD = 'output.size'; const DEFAULT_TERMS_SIZE = 10; const PRINTABLE_PDF_JOBTYPE = 'printable_pdf'; @@ -71,16 +69,12 @@ const getAppStatuses = (buckets: StatusByAppBucket[]) => function getAggStats(aggs: AggregationResultBuckets): Partial { const { buckets: jobBuckets } = aggs[JOB_TYPES_KEY] as AggregationBuckets; const jobTypes = jobBuckets.reduce((accum: JobTypes, bucket) => { - const { key, doc_count: count, isDeprecated, sizeMax, sizeMin, sizeAvg } = bucket; + const { key, doc_count: count, isDeprecated, sizes } = bucket; const deprecatedCount = isDeprecated?.doc_count; const total: Omit = { total: count, deprecated: deprecatedCount, - output_size: { - max: sizeMax?.value ?? null, - min: sizeMin?.value ?? null, - avg: sizeAvg?.value ?? null, - }, + sizes: sizes?.values, }; return { ...accum, [key]: total }; }, {} as JobTypes); @@ -111,11 +105,7 @@ function getAggStats(aggs: AggregationResultBuckets): Partial { _all: all, status: statusTypes, statuses: statusByApp, - output_size: { - max: get(aggs[OUTPUT_SIZE_MAX_KEY], 'value') ?? null, - min: get(aggs[OUTPUT_SIZE_MIN_KEY], 'value') ?? null, - avg: get(aggs[OUTPUT_SIZE_AVG_KEY], 'value') ?? null, - }, + output_size: get(aggs[OUTPUT_SIZES_KEY], 'values') ?? undefined, ...jobTypes, }; } @@ -155,19 +145,6 @@ export async function getReportingUsage( exportTypesRegistry: ExportTypesRegistry ): Promise { const reportingIndex = config.get('index'); - - const outputSizeAggs = { - [OUTPUT_SIZE_MIN_KEY]: { - min: { field: OUTPUT_SIZE_FIELD }, - }, - [OUTPUT_SIZE_MAX_KEY]: { - max: { field: OUTPUT_SIZE_FIELD }, - }, - [OUTPUT_SIZE_AVG_KEY]: { - avg: { field: OUTPUT_SIZE_FIELD }, - }, - }; - const params = { index: `${reportingIndex}-*`, filterPath: 'aggregations.*.buckets', @@ -186,7 +163,9 @@ export async function getReportingUsage( terms: { field: JOB_TYPES_FIELD, size: DEFAULT_TERMS_SIZE }, aggs: { isDeprecated: { filter: { term: { [OBJECT_TYPE_DEPRECATED_KEY]: true } } }, - ...outputSizeAggs, + [OUTPUT_SIZES_KEY]: { + percentiles: { field: OUTPUT_SIZES_FIELD }, + }, }, }, @@ -212,7 +191,9 @@ export async function getReportingUsage( filter: { term: { jobtype: PRINTABLE_PDF_JOBTYPE } }, aggs: { pdf: { terms: { field: LAYOUT_TYPES_FIELD, size: DEFAULT_TERMS_SIZE } } }, }, - ...outputSizeAggs, + [OUTPUT_SIZES_KEY]: { + percentiles: { field: OUTPUT_SIZES_FIELD }, + }, }, }, }, diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index be22071e2f7f7..9580ddb935dfb 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -14,7 +14,7 @@ import { LayoutCounts, RangeStats, ReportingUsageType, - SizeMetrics, + SizePercentiles, } from './types'; const appCountsSchema: MakeSchemaFrom = { @@ -40,17 +40,21 @@ const byAppCountsSchema: MakeSchemaFrom = { printable_pdf_v2: appCountsSchema, }; -const sizesSchema: MakeSchemaFrom = { - max: { type: 'long' }, - min: { type: 'long' }, - avg: { type: 'long' }, +const sizesSchema: MakeSchemaFrom = { + '1.0': { type: 'long' }, + '5.0': { type: 'long' }, + '25.0': { type: 'long' }, + '50.0': { type: 'long' }, + '75.0': { type: 'long' }, + '95.0': { type: 'long' }, + '99.0': { type: 'long' }, }; const availableTotalSchema: MakeSchemaFrom = { available: { type: 'boolean' }, total: { type: 'long' }, deprecated: { type: 'long' }, - output_size: sizesSchema, + sizes: sizesSchema, app: appCountsSchema, layout: layoutCountsSchema, }; diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index a40e187abdd72..289f46deb57aa 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -5,10 +5,18 @@ * 2.0. */ +export interface SizePercentiles { + '1.0': number | null; + '5.0': number | null; + '25.0': number | null; + '50.0': number | null; + '75.0': number | null; + '95.0': number | null; + '99.0': number | null; +} + interface SizeStats { - sizeMin?: { value: number | null }; - sizeMax?: { value: number | null }; - sizeAvg?: { value: number | null }; + sizes?: { values: SizePercentiles }; } export interface KeyCountBucket extends SizeStats { @@ -63,17 +71,11 @@ export interface SearchResponse { }; } -export interface SizeMetrics { - max: number | null; - min: number | null; - avg: number | null; -} - export interface AvailableTotal { available: boolean; total: number; deprecated?: number; - output_size: SizeMetrics; + sizes?: SizePercentiles; app?: { search?: number; dashboard?: number; @@ -124,7 +126,7 @@ export type RangeStats = JobTypes & { _all: number; status: StatusCounts; statuses?: StatusByAppCounts; - output_size: SizeMetrics; + output_size?: SizePercentiles; }; export type ReportingUsageType = RangeStats & { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index a7298008c1ba9..98c81a6f9c677 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -4051,15 +4051,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4106,15 +4118,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4161,15 +4185,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4216,15 +4252,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4271,15 +4319,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4326,15 +4386,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -4381,15 +4453,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5033,13 +5117,25 @@ }, "output_size": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5066,15 +5162,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5121,15 +5229,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5176,15 +5296,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5231,15 +5363,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5286,15 +5430,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5341,15 +5497,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -5396,15 +5564,27 @@ "deprecated": { "type": "long" }, - "output_size": { + "sizes": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } @@ -6048,13 +6228,25 @@ }, "output_size": { "properties": { - "max": { + "1.0": { "type": "long" }, - "min": { + "5.0": { "type": "long" }, - "avg": { + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { "type": "long" } } From 55580255a0eecde61c7300ee9cf8383b532d857d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 22 Sep 2021 14:00:29 -0700 Subject: [PATCH 10/13] DocCount interface --- .../plugins/reporting/server/usage/types.ts | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 289f46deb57aa..856d3ad10cb26 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -15,49 +15,47 @@ export interface SizePercentiles { '99.0': number | null; } +interface DocCount { + doc_count: number; +} + interface SizeStats { sizes?: { values: SizePercentiles }; } -export interface KeyCountBucket extends SizeStats { +export interface KeyCountBucket extends DocCount, SizeStats { key: string; - doc_count: number; - isDeprecated?: { - doc_count: number; - }; + isDeprecated?: DocCount; } export interface AggregationBuckets { buckets: KeyCountBucket[]; } -export interface StatusByAppBucket { +export interface StatusByAppBucket extends DocCount { key: string; - doc_count: number; jobTypes: { - buckets: Array<{ - doc_count: number; - key: string; - appNames: AggregationBuckets; - }>; + buckets: Array< + { + key: string; + appNames: AggregationBuckets; + } & DocCount + >; }; } -export interface AggregationResultBuckets extends SizeStats { +export interface AggregationResultBuckets extends DocCount, SizeStats { jobTypes?: AggregationBuckets; layoutTypes: { - doc_count: number; pdf?: AggregationBuckets; - }; + } & DocCount; objectTypes: { - doc_count: number; pdf?: AggregationBuckets; - }; + } & DocCount; statusTypes: AggregationBuckets; statusByApp: { buckets: StatusByAppBucket[]; }; - doc_count: number; } export interface SearchResponse { From 9d6b579bfa3afd81e51ae76b58b52a6243b06bf6 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 22 Sep 2021 14:08:19 -0700 Subject: [PATCH 11/13] fix tests --- .../reporting_usage_collector.test.ts.snap | 700 ++++++++---------- .../server/usage/get_export_stats.test.ts | 75 +- 2 files changed, 368 insertions(+), 407 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 96b352b9977f1..2ced5483cc4c5 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -37,14 +37,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { "type": "long", }, - "max": Object { + "95.0": Object { "type": "long", }, - "min": Object { + "99.0": Object { "type": "long", }, }, @@ -84,14 +96,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { "type": "long", }, - "max": Object { + "75.0": Object { "type": "long", }, - "min": Object { + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -140,14 +164,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { "type": "long", }, - "max": Object { + "75.0": Object { "type": "long", }, - "min": Object { + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -187,14 +223,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -234,14 +282,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -285,14 +345,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -332,14 +404,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -382,14 +466,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -429,14 +525,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { "type": "long", }, - "max": Object { + "25.0": Object { "type": "long", }, - "min": Object { + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -476,14 +584,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -492,13 +612,25 @@ Object { }, }, "output_size": Object { - "avg": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { "type": "long", }, - "max": Object { + "95.0": Object { "type": "long", }, - "min": Object { + "99.0": Object { "type": "long", }, }, @@ -534,14 +666,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -581,14 +725,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { "type": "long", }, - "max": Object { + "95.0": Object { "type": "long", }, - "min": Object { + "99.0": Object { "type": "long", }, }, @@ -1117,13 +1273,25 @@ Object { }, }, "output_size": Object { - "avg": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { "type": "long", }, - "max": Object { + "95.0": Object { "type": "long", }, - "min": Object { + "99.0": Object { "type": "long", }, }, @@ -1159,14 +1327,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { "type": "long", }, - "max": Object { + "50.0": Object { "type": "long", }, - "min": Object { + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -1206,14 +1386,26 @@ Object { "type": "long", }, }, - "output_size": Object { - "avg": Object { + "sizes": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { "type": "long", }, - "max": Object { + "5.0": Object { "type": "long", }, - "min": Object { + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { "type": "long", }, }, @@ -1761,11 +1953,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -1782,11 +1970,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 9, @@ -1806,11 +1990,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 4, }, "csv_searchsource": Object { @@ -1827,11 +2007,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1848,11 +2024,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "enabled": true, @@ -1871,11 +2043,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -1892,11 +2060,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 9, @@ -1914,11 +2078,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 4, }, "csv_searchsource": Object { @@ -1935,11 +2095,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 5, }, "csv_searchsource_immediate": Object { @@ -1956,18 +2112,10 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -1982,11 +2130,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2003,11 +2147,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2025,11 +2165,7 @@ Object { }, }, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2044,11 +2180,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2065,11 +2197,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2105,11 +2233,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2126,11 +2250,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 0, @@ -2150,11 +2270,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2171,11 +2287,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2192,11 +2304,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "enabled": true, @@ -2215,11 +2323,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2236,11 +2340,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 0, @@ -2258,11 +2358,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2279,11 +2375,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2300,18 +2392,10 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2326,11 +2410,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2347,11 +2427,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2360,11 +2436,7 @@ Object { }, "statuses": Object {}, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -2379,11 +2451,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2400,11 +2468,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2431,11 +2495,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 1, }, "PNGV2": Object { @@ -2452,11 +2512,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 4, @@ -2476,11 +2532,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 1, }, "csv_searchsource": Object { @@ -2497,11 +2549,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2518,11 +2566,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "enabled": true, @@ -2541,11 +2585,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 1, }, "PNGV2": Object { @@ -2562,11 +2602,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 4, @@ -2584,11 +2620,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 1, }, "csv_searchsource": Object { @@ -2605,11 +2637,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2626,18 +2654,10 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 1, @@ -2652,11 +2672,7 @@ Object { "preserve_layout": 2, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 2, }, "printable_pdf_v2": Object { @@ -2673,11 +2689,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2697,11 +2709,7 @@ Object { }, }, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 1, @@ -2716,11 +2724,7 @@ Object { "preserve_layout": 2, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "total": 2, }, "printable_pdf_v2": Object { @@ -2737,11 +2741,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -2779,11 +2779,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": 37748, - "max": 37748, - "min": 37748, - }, + "output_size": undefined, "total": 1, }, "PNGV2": Object { @@ -2800,11 +2796,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 11, @@ -2824,11 +2816,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": 231, - "max": 231, - "min": 231, - }, + "output_size": undefined, "total": 1, }, "csv_searchsource": Object { @@ -2845,11 +2833,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": 3684.6666666666665, - "max": 7557, - "min": 204, - }, + "output_size": undefined, "total": 3, }, "csv_searchsource_immediate": Object { @@ -2866,11 +2850,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "enabled": true, @@ -2889,11 +2869,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2910,11 +2886,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "_all": 0, @@ -2932,11 +2904,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2953,11 +2921,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2974,18 +2938,10 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -3000,11 +2956,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -3021,11 +2973,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { @@ -3034,11 +2982,7 @@ Object { }, "statuses": Object {}, }, - "output_size": Object { - "avg": 365084.75, - "max": 1713303, - "min": 204, - }, + "output_size": undefined, "printable_pdf": Object { "app": Object { "canvas workpad": 0, @@ -3053,11 +2997,7 @@ Object { "preserve_layout": 5, "print": 1, }, - "output_size": Object { - "avg": 957215, - "max": 1713303, - "min": 43226, - }, + "output_size": undefined, "total": 6, }, "printable_pdf_v2": Object { @@ -3074,11 +3014,7 @@ Object { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, }, "status": Object { diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index 396d792f7cc5e..91d88cb18a447 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -11,6 +11,15 @@ import { getExportTypesHandler } from './get_export_type_handler'; import { FeatureAvailabilityMap } from './types'; let featureMap: FeatureAvailabilityMap; +const sizesAggResponse = { + '1.0': 5093470.0, + '5.0': 5093470.0, + '25.0': 5093470.0, + '50.0': 8514532.0, + '75.0': 1.1935594e7, + '95.0': 1.1935594e7, + '99.0': 1.1935594e7, +}; beforeEach(() => { featureMap = { PNG: true, csv: true, csv_searchsource: true, printable_pdf: true }; @@ -67,18 +76,18 @@ test('Model of job status and status-by-pdf-app', () => { test('Model of jobTypes', () => { const result = getExportStats( { - PNG: { available: true, total: 3, sizes: { max: 999, min: 333, avg: 666 } }, + PNG: { available: true, total: 3, sizes: sizesAggResponse }, printable_pdf: { available: true, total: 3, - sizes: { max: 999, min: 333, avg: 666 }, + sizes: sizesAggResponse, app: { dashboard: 0, visualization: 0, 'canvas workpad': 3 }, layout: { preserve_layout: 3, print: 0 }, }, csv_searchsource: { available: true, total: 3, - sizes: { max: 999, min: 333, avg: 666 }, + sizes: sizesAggResponse, }, }, featureMap, @@ -101,9 +110,13 @@ test('Model of jobTypes', () => { "print": 0, }, "output_size": Object { - "avg": 666, - "max": 999, - "min": 333, + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, }, "total": 3, } @@ -123,11 +136,7 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, - "output_size": Object { - "avg": null, - "max": null, - "min": null, - }, + "sizes": undefined, "total": 0, } `); @@ -147,9 +156,13 @@ test('Model of jobTypes', () => { "print": 0, }, "output_size": Object { - "avg": 666, - "max": 999, - "min": 333, + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, }, "total": 3, } @@ -170,9 +183,13 @@ test('Model of jobTypes', () => { "print": 0, }, "output_size": Object { - "avg": 666, - "max": 999, - "min": 333, + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, }, "total": 3, } @@ -186,7 +203,7 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { available: true, total: 15, deprecated: 5, - sizes: { min: 111, max: 333, avg: 222 }, + sizes: sizesAggResponse, }, }, featureMap, @@ -208,9 +225,13 @@ test('PNG counts, provided count of deprecated jobs explicitly', () => { "print": 0, }, "output_size": Object { - "avg": 222, - "max": 333, - "min": 111, + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, }, "total": 15, } @@ -224,7 +245,7 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => available: true, total: 15, deprecated: 0, - sizes: { min: 111, max: 333, avg: 222 }, + sizes: sizesAggResponse, }, }, featureMap, @@ -246,9 +267,13 @@ test('CSV counts, provides all jobs implicitly deprecated due to jobtype', () => "print": 0, }, "output_size": Object { - "avg": 222, - "max": 333, - "min": 111, + "1.0": 5093470, + "25.0": 5093470, + "5.0": 5093470, + "50.0": 8514532, + "75.0": 11935594, + "95.0": 11935594, + "99.0": 11935594, }, "total": 15, } From 867bcb9d011834e1c3b89e4babb77c429e92d7e5 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 22 Sep 2021 14:11:32 -0700 Subject: [PATCH 12/13] Update get_export_stats.ts --- x-pack/plugins/reporting/server/usage/get_export_stats.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index 34ba25fc4ce20..72c09f08017a1 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -13,7 +13,6 @@ const jobTypeIsDeprecated = (jobType: string) => DEPRECATED_JOB_TYPES.includes(j const defaultTotalsForFeature: Omit = { total: 0, deprecated: 0, - sizes: undefined, app: { 'canvas workpad': 0, search: 0, visualization: 0, dashboard: 0 }, layout: { canvas: 0, print: 0, preserve_layout: 0 }, }; From 2f32ac498cb6cc31bd02b0751b65c02656992e6d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Thu, 23 Sep 2021 14:11:40 -0700 Subject: [PATCH 13/13] update snapshots --- .../reporting_usage_collector.test.ts.snap | 42 ------------------- .../server/usage/get_export_stats.test.ts | 1 - 2 files changed, 43 deletions(-) diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 2ced5483cc4c5..2017ae0be59c7 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -1953,7 +1953,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -1970,7 +1969,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 9, @@ -2024,7 +2022,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "enabled": true, @@ -2043,7 +2040,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2060,7 +2056,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 9, @@ -2112,7 +2107,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "output_size": undefined, @@ -2130,7 +2124,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2147,7 +2140,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2180,7 +2172,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2197,7 +2188,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2233,7 +2223,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2250,7 +2239,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 0, @@ -2270,7 +2258,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2287,7 +2274,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2304,7 +2290,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "enabled": true, @@ -2323,7 +2308,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2340,7 +2324,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 0, @@ -2358,7 +2341,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2375,7 +2357,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2392,7 +2373,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "output_size": undefined, @@ -2410,7 +2390,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2427,7 +2406,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2451,7 +2429,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2468,7 +2445,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2512,7 +2488,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 4, @@ -2549,7 +2524,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2566,7 +2540,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "enabled": true, @@ -2602,7 +2575,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 4, @@ -2637,7 +2609,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2654,7 +2625,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "output_size": undefined, @@ -2689,7 +2659,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2741,7 +2710,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -2796,7 +2764,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 11, @@ -2850,7 +2817,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "enabled": true, @@ -2869,7 +2835,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "PNGV2": Object { @@ -2886,7 +2851,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "_all": 0, @@ -2904,7 +2868,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource": Object { @@ -2921,7 +2884,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "csv_searchsource_immediate": Object { @@ -2938,7 +2900,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "output_size": undefined, @@ -2956,7 +2917,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "printable_pdf_v2": Object { @@ -2973,7 +2933,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { @@ -3014,7 +2973,6 @@ Object { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, }, "status": Object { diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts index 91d88cb18a447..f74e176e6f21d 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.test.ts @@ -136,7 +136,6 @@ test('Model of jobTypes', () => { "preserve_layout": 0, "print": 0, }, - "sizes": undefined, "total": 0, } `);