From 23ae82379ef083f21de43e74b33f5cdf8891cebb Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 27 Oct 2020 15:41:44 +0200 Subject: [PATCH 1/4] remove number type and support all es number types --- packages/kbn-telemetry-tools/GUIDELINE.md | 9 +-------- .../src/tools/manage_schema.ts | 18 ++++++++---------- .../data/server/search/collectors/register.ts | 4 ++-- src/plugins/telemetry/schema/oss_plugins.json | 4 ++-- src/plugins/usage_collection/README.md | 2 +- .../server/collector/collector.ts | 11 +++-------- .../usage_collection/server/collector/index.ts | 1 + .../security_usage_collector.ts | 2 +- .../schema/xpack_plugins.json | 2 +- 9 files changed, 20 insertions(+), 33 deletions(-) diff --git a/packages/kbn-telemetry-tools/GUIDELINE.md b/packages/kbn-telemetry-tools/GUIDELINE.md index e7d09babbf9e2..a22196bb5dc74 100644 --- a/packages/kbn-telemetry-tools/GUIDELINE.md +++ b/packages/kbn-telemetry-tools/GUIDELINE.md @@ -148,14 +148,7 @@ usageCollection.makeUsageCollector({ Any field property in the schema accepts a `type` field. By default the type is `object` which accepts nested properties under it. Currently we accept the following property types: ``` -AllowedSchemaTypes = - | 'keyword' - | 'text' - | 'number' - | 'boolean' - | 'long' - | 'date' - | 'float'; +'long', 'integer', 'short', 'byte', 'double', 'float', 'keyword', 'text', 'boolean', 'date' ``` diff --git a/packages/kbn-telemetry-tools/src/tools/manage_schema.ts b/packages/kbn-telemetry-tools/src/tools/manage_schema.ts index 7721492fdb691..e2bfca34a6487 100644 --- a/packages/kbn-telemetry-tools/src/tools/manage_schema.ts +++ b/packages/kbn-telemetry-tools/src/tools/manage_schema.ts @@ -19,14 +19,9 @@ import { ParsedUsageCollection } from './ts_parser'; -export type AllowedSchemaTypes = - | 'keyword' - | 'text' - | 'number' - | 'boolean' - | 'long' - | 'date' - | 'float'; +export type AllowedSchemaNumberTypes = 'long' | 'integer' | 'short' | 'byte' | 'double' | 'float'; + +export type AllowedSchemaTypes = AllowedSchemaNumberTypes | 'keyword' | 'text' | 'boolean' | 'date'; export function compatibleSchemaTypes(type: AllowedSchemaTypes | 'array') { switch (type) { @@ -36,9 +31,12 @@ export function compatibleSchemaTypes(type: AllowedSchemaTypes | 'array') { return 'string'; case 'boolean': return 'boolean'; - case 'number': - case 'float': case 'long': + case 'integer': + case 'short': + case 'byte': + case 'double': + case 'float': return 'number'; case 'array': return 'array'; diff --git a/src/plugins/data/server/search/collectors/register.ts b/src/plugins/data/server/search/collectors/register.ts index ab0ea93edd49e..00e81c42f4d3b 100644 --- a/src/plugins/data/server/search/collectors/register.ts +++ b/src/plugins/data/server/search/collectors/register.ts @@ -37,8 +37,8 @@ export async function registerUsageCollector( isReady: () => true, fetch: fetchProvider(context.config.legacy.globalConfig$), schema: { - successCount: { type: 'number' }, - errorCount: { type: 'number' }, + successCount: { type: 'long' }, + errorCount: { type: 'long' }, averageDuration: { type: 'long' }, }, }); diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 160e99a40790c..17d9ad5384b44 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -16,10 +16,10 @@ "search": { "properties": { "successCount": { - "type": "number" + "type": "long" }, "errorCount": { - "type": "number" + "type": "long" }, "averageDuration": { "type": "long" diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md index 430241cbe0a05..f68b683de387b 100644 --- a/src/plugins/usage_collection/README.md +++ b/src/plugins/usage_collection/README.md @@ -138,7 +138,7 @@ The `schema` field is a proscribed data model assists with detecting changes in The `AllowedSchemaTypes` is the list of allowed schema types for the usage fields getting reported: ``` -'keyword', 'text', 'number', 'boolean', 'long', 'date', 'float' +'long', 'integer', 'short', 'byte', 'double', 'float', 'keyword', 'text', 'boolean', 'date' ``` ### Arrays diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 951418d448cbd..73febc0183fc5 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -27,14 +27,9 @@ import { export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; -export type AllowedSchemaTypes = - | 'keyword' - | 'text' - | 'number' - | 'boolean' - | 'long' - | 'date' - | 'float'; +export type AllowedSchemaNumberTypes = 'long' | 'integer' | 'short' | 'byte' | 'double' | 'float'; + +export type AllowedSchemaTypes = AllowedSchemaNumberTypes | 'keyword' | 'text' | 'boolean' | 'date'; export interface SchemaField { type: string; diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index da85f9ab181c9..2f8be884a8a7b 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -21,6 +21,7 @@ export { CollectorSet } from './collector_set'; export { Collector, AllowedSchemaTypes, + AllowedSchemaNumberTypes, SchemaField, MakeSchemaFrom, CollectorOptions, diff --git a/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts index 90483d7c0a4d5..cf3c787e6b0db 100644 --- a/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts +++ b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts @@ -59,7 +59,7 @@ export function registerSecurityUsageCollector({ usageCollection, config, licens type: 'boolean', }, authProviderCount: { - type: 'number', + type: 'long', }, enabledAuthProviders: { type: 'array', diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 396a06205eaa9..ec68c740b0d0d 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3240,7 +3240,7 @@ "type": "boolean" }, "authProviderCount": { - "type": "number" + "type": "long" }, "enabledAuthProviders": { "type": "array", From 9807b16211de3c258c87b53acda821d3e93ea957 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 27 Oct 2020 15:52:36 +0200 Subject: [PATCH 2/4] update docs --- src/plugins/usage_collection/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md index f68b683de387b..5a853972d34a8 100644 --- a/src/plugins/usage_collection/README.md +++ b/src/plugins/usage_collection/README.md @@ -171,7 +171,7 @@ export const myCollector = makeUsageCollector({ }, some_obj: { total: { - type: 'number', + type: 'long', }, }, some_array: { @@ -182,7 +182,7 @@ export const myCollector = makeUsageCollector({ type: 'array', items: { total: { - type: 'number', + type: 'long', }, }, }, From 1d89ce1475bc79f98eaeeabd63f929596069d44c Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 27 Oct 2020 18:41:57 +0200 Subject: [PATCH 3/4] fix tests --- .../src/tools/__fixture__/mock_schema.json | 12 ++++++------ ...d_indexed_interface_with_not_matching_schema.ts | 2 +- ...parsed_schema_defined_with_spreads_collector.ts | 2 +- .../tools/__fixture__/parsed_working_collector.ts | 12 ++++++------ .../__snapshots__/extract_collectors.test.ts.snap | 14 +++++++------- .../src/tools/check_collector__integrity.test.ts | 6 +++--- .../schema_defined_with_spreads_collector.ts | 2 +- .../telemetry_collectors/working_collector.ts | 12 ++++++------ 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json b/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json index 51e5df9bf7dc0..62c0b1ff7eb15 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json @@ -8,16 +8,16 @@ "my_index_signature_prop": { "properties": { "avg": { - "type": "number" + "type": "long" }, "count": { - "type": "number" + "type": "long" }, "max": { - "type": "number" + "type": "long" }, "min": { - "type": "number" + "type": "long" } } }, @@ -27,7 +27,7 @@ "my_objects": { "properties": { "total": { - "type": "number" + "type": "long" }, "type": { "type": "boolean" @@ -39,7 +39,7 @@ "items": { "properties": { "total": { - "type": "number" + "type": "long" }, "type": { "type": "boolean" diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_indexed_interface_with_not_matching_schema.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_indexed_interface_with_not_matching_schema.ts index 83866a2b6afec..109fc045b6ee0 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_indexed_interface_with_not_matching_schema.ts +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_indexed_interface_with_not_matching_schema.ts @@ -28,7 +28,7 @@ export const parsedIndexedInterfaceWithNoMatchingSchema: ParsedUsageCollection = value: { something: { count_1: { - type: 'number', + type: 'long', }, }, }, diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts index 833344fa368b0..4a1a622e23f36 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts @@ -34,7 +34,7 @@ export const parsedSchemaDefinedWithSpreadsCollector: ParsedUsageCollection = [ }, my_objects: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean', diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts index acf984b7d10ee..39c820d500a9d 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts @@ -34,21 +34,21 @@ export const parsedWorkingCollector: ParsedUsageCollection = [ }, my_index_signature_prop: { avg: { - type: 'number', + type: 'long', }, count: { - type: 'number', + type: 'long', }, max: { - type: 'number', + type: 'long', }, min: { - type: 'number', + type: 'long', }, }, my_objects: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean', @@ -58,7 +58,7 @@ export const parsedWorkingCollector: ParsedUsageCollection = [ type: 'array', items: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean' }, }, diff --git a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap index 4725be77533af..171b5d10b02a5 100644 --- a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap +++ b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap @@ -176,7 +176,7 @@ Array [ }, "my_objects": Object { "total": Object { - "type": "number", + "type": "long", }, "type": Object { "type": "boolean", @@ -248,7 +248,7 @@ Array [ "my_array": Object { "items": Object { "total": Object { - "type": "number", + "type": "long", }, "type": Object { "type": "boolean", @@ -258,21 +258,21 @@ Array [ }, "my_index_signature_prop": Object { "avg": Object { - "type": "number", + "type": "long", }, "count": Object { - "type": "number", + "type": "long", }, "max": Object { - "type": "number", + "type": "long", }, "min": Object { - "type": "number", + "type": "long", }, }, "my_objects": Object { "total": Object { - "type": "number", + "type": "long", }, "type": Object { "type": "boolean", diff --git a/packages/kbn-telemetry-tools/src/tools/check_collector__integrity.test.ts b/packages/kbn-telemetry-tools/src/tools/check_collector__integrity.test.ts index a101210185a63..b6ea9d49cf6d0 100644 --- a/packages/kbn-telemetry-tools/src/tools/check_collector__integrity.test.ts +++ b/packages/kbn-telemetry-tools/src/tools/check_collector__integrity.test.ts @@ -44,7 +44,7 @@ describe('checkMatchingMapping', () => { it('returns diff on mismatching parsedCollections and stored mapping', async () => { const mockSchema = await parseJsonFile('mock_schema.json'); const malformedParsedCollector = cloneDeep(parsedWorkingCollector); - const fieldMapping = { type: 'number' }; + const fieldMapping = { type: 'long' }; malformedParsedCollector[1].schema.value.flat = fieldMapping; const diffs = checkMatchingMapping([malformedParsedCollector], mockSchema); @@ -61,9 +61,9 @@ describe('checkMatchingMapping', () => { const mockSchema = await parseJsonFile('mock_schema.json'); const malformedParsedCollector = cloneDeep(parsedWorkingCollector); const collectorName = 'New Collector in town!'; - const collectorMapping = { some_usage: { type: 'number' } }; + const collectorMapping = { some_usage: { type: 'long' } }; malformedParsedCollector[1].collectorName = collectorName; - malformedParsedCollector[1].schema.value = { some_usage: { type: 'number' } }; + malformedParsedCollector[1].schema.value = { some_usage: { type: 'long' } }; const diffs = checkMatchingMapping([malformedParsedCollector], mockSchema); expect(diffs).toEqual({ diff --git a/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts b/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts index af9fef0bbd297..791c366199d74 100644 --- a/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts +++ b/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts @@ -49,7 +49,7 @@ const someSchema: MakeSchemaFrom> = { const someOtherSchema: MakeSchemaFrom> = { my_objects: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean' }, }, diff --git a/src/fixtures/telemetry_collectors/working_collector.ts b/src/fixtures/telemetry_collectors/working_collector.ts index 0a3bf49638a7b..902b411f95a41 100644 --- a/src/fixtures/telemetry_collectors/working_collector.ts +++ b/src/fixtures/telemetry_collectors/working_collector.ts @@ -85,7 +85,7 @@ export const myCollector = makeUsageCollector({ }, my_objects: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean' }, }, @@ -93,17 +93,17 @@ export const myCollector = makeUsageCollector({ type: 'array', items: { total: { - type: 'number', + type: 'long', }, type: { type: 'boolean' }, }, }, my_str_array: { type: 'array', items: { type: 'keyword' } }, my_index_signature_prop: { - count: { type: 'number' }, - avg: { type: 'number' }, - max: { type: 'number' }, - min: { type: 'number' }, + count: { type: 'long' }, + avg: { type: 'long' }, + max: { type: 'long' }, + min: { type: 'long' }, }, }, }); From 00650806031caafe8d93cd2620da7be3a992da05 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Thu, 29 Oct 2020 13:35:59 +0200 Subject: [PATCH 4/4] long -> float for averages --- .../kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json | 2 +- .../src/tools/__fixture__/parsed_working_collector.ts | 2 +- .../src/tools/__snapshots__/extract_collectors.test.ts.snap | 2 +- src/fixtures/telemetry_collectors/working_collector.ts | 2 +- src/plugins/data/server/search/collectors/register.ts | 2 +- src/plugins/telemetry/schema/oss_plugins.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json b/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json index 62c0b1ff7eb15..a385cd6798365 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json @@ -8,7 +8,7 @@ "my_index_signature_prop": { "properties": { "avg": { - "type": "long" + "type": "float" }, "count": { "type": "long" diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts index 39c820d500a9d..ef6227cf35c37 100644 --- a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts @@ -34,7 +34,7 @@ export const parsedWorkingCollector: ParsedUsageCollection = [ }, my_index_signature_prop: { avg: { - type: 'long', + type: 'float', }, count: { type: 'long', diff --git a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap index 171b5d10b02a5..fe589be7993d0 100644 --- a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap +++ b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap @@ -258,7 +258,7 @@ Array [ }, "my_index_signature_prop": Object { "avg": Object { - "type": "long", + "type": "float", }, "count": Object { "type": "long", diff --git a/src/fixtures/telemetry_collectors/working_collector.ts b/src/fixtures/telemetry_collectors/working_collector.ts index 902b411f95a41..f9cb3bb568673 100644 --- a/src/fixtures/telemetry_collectors/working_collector.ts +++ b/src/fixtures/telemetry_collectors/working_collector.ts @@ -101,7 +101,7 @@ export const myCollector = makeUsageCollector({ my_str_array: { type: 'array', items: { type: 'keyword' } }, my_index_signature_prop: { count: { type: 'long' }, - avg: { type: 'long' }, + avg: { type: 'float' }, max: { type: 'long' }, min: { type: 'long' }, }, diff --git a/src/plugins/data/server/search/collectors/register.ts b/src/plugins/data/server/search/collectors/register.ts index 00e81c42f4d3b..5db4f52169350 100644 --- a/src/plugins/data/server/search/collectors/register.ts +++ b/src/plugins/data/server/search/collectors/register.ts @@ -39,7 +39,7 @@ export async function registerUsageCollector( schema: { successCount: { type: 'long' }, errorCount: { type: 'long' }, - averageDuration: { type: 'long' }, + averageDuration: { type: 'float' }, }, }); usageCollection.registerCollector(collector); diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 17d9ad5384b44..c840cbe8fc94d 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -22,7 +22,7 @@ "type": "long" }, "averageDuration": { - "type": "long" + "type": "float" } } },