[7.x] [Usage Collection] [schema] Explicit "array" definition (#78141) (

#78618) Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
elastic · Sep 28, 2020 · 8dd6dcb · 8dd6dcb
1 parent b4add7b
commit 8dd6dcb
Show file tree

Hide file tree

Showing 19 changed files with 362 additions and 240 deletions.
diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json b/packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json
@@ -35,16 +35,19 @@
           }
         },
         "my_array": {
-          "properties": {
-            "total": {
-              "type": "number"
-            },
-            "type": {
-              "type": "boolean"
+          "type": "array",
+          "items": {
+            "properties": {
+              "total": {
+                "type": "number"
+              },
+              "type": {
+                "type": "boolean"
+              }
             }
           }
         },
-        "my_str_array": { "type": "keyword" }
+        "my_str_array": { "type": "array", "items": { "type": "keyword" } }
       }
     }
   }

diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts
@@ -55,12 +55,15 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
           },
         },
         my_array: {
-          total: {
-            type: 'number',
+          type: 'array',
+          items: {
+            total: {
+              type: 'number',
+            },
+            type: { type: 'boolean' },
           },
-          type: { type: 'boolean' },
         },
-        my_str_array: { type: 'keyword' },
+        my_str_array: { type: 'array', items: { type: 'keyword' } },
       },
     },
     fetch: {
@@ -91,18 +94,22 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
           },
         },
         my_array: {
-          total: {
-            kind: SyntaxKind.NumberKeyword,
-            type: 'NumberKeyword',
-          },
-          type: {
-            kind: SyntaxKind.BooleanKeyword,
-            type: 'BooleanKeyword',
+          items: {
+            total: {
+              kind: SyntaxKind.NumberKeyword,
+              type: 'NumberKeyword',
+            },
+            type: {
+              kind: SyntaxKind.BooleanKeyword,
+              type: 'BooleanKeyword',
+            },
           },
         },
         my_str_array: {
-          kind: SyntaxKind.StringKeyword,
-          type: 'StringKeyword',
+          items: {
+            kind: SyntaxKind.StringKeyword,
+            type: 'StringKeyword',
+          },
         },
       },
     },

diff --git a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap
diff --git a/packages/kbn-telemetry-tools/src/tools/manage_schema.ts b/packages/kbn-telemetry-tools/src/tools/manage_schema.ts
@@ -28,7 +28,7 @@ export type AllowedSchemaTypes =
   | 'date'
   | 'float';
 
-export function compatibleSchemaTypes(type: AllowedSchemaTypes) {
+export function compatibleSchemaTypes(type: AllowedSchemaTypes | 'array') {
   switch (type) {
     case 'keyword':
     case 'text':
@@ -40,6 +40,8 @@ export function compatibleSchemaTypes(type: AllowedSchemaTypes) {
     case 'float':
     case 'long':
       return 'number';
+    case 'array':
+      return 'array';
     default:
       throw new Error(`Unknown schema type ${type}`);
   }
@@ -66,10 +68,22 @@ export function isObjectMapping(entity: any) {
   return false;
 }
 
+function isArrayMapping(entity: any): entity is { type: 'array'; items: object } {
+  return typeof entity === 'object' && entity.type === 'array' && typeof entity.items === 'object';
+}
+
+function getValueMapping(value: any) {
+  return isObjectMapping(value) ? transformToEsMapping(value) : value;
+}
+
 function transformToEsMapping(usageMappingValue: any) {
   const fieldMapping: any = { properties: {} };
   for (const [key, value] of Object.entries(usageMappingValue)) {
-    fieldMapping.properties[key] = isObjectMapping(value) ? transformToEsMapping(value) : value;
+    if (isArrayMapping(value)) {
+      fieldMapping.properties[key] = { ...value, items: getValueMapping(value.items) };
+    } else {
+      fieldMapping.properties[key] = getValueMapping(value);
+    }
   }
   return fieldMapping;
 }

diff --git a/packages/kbn-telemetry-tools/src/tools/serializer.test.ts b/packages/kbn-telemetry-tools/src/tools/serializer.test.ts
@@ -84,8 +84,8 @@ describe('getDescriptor', () => {
     expect(descriptor).toEqual({
       prop1: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
       prop2: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
-      prop3: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' },
-      prop4: { kind: TelemetryKinds.Date, type: 'Date' },
+      prop3: { items: { kind: TelemetryKinds.MomentDate, type: 'MomentDate' } },
+      prop4: { items: { kind: TelemetryKinds.Date, type: 'Date' } },
     });
   });
 

diff --git a/packages/kbn-telemetry-tools/src/tools/serializer.ts b/packages/kbn-telemetry-tools/src/tools/serializer.ts
@@ -139,7 +139,7 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
   }
 
   if (ts.isArrayTypeNode(node)) {
-    return getDescriptor(node.elementType, program);
+    return { items: getDescriptor(node.elementType, program) };
   }
 
   if (ts.isLiteralTypeNode(node)) {

diff --git a/src/fixtures/telemetry_collectors/working_collector.ts b/src/fixtures/telemetry_collectors/working_collector.ts
@@ -90,12 +90,15 @@ export const myCollector = makeUsageCollector<Usage>({
       type: { type: 'boolean' },
     },
     my_array: {
-      total: {
-        type: 'number',
+      type: 'array',
+      items: {
+        total: {
+          type: 'number',
+        },
+        type: { type: 'boolean' },
       },
-      type: { type: 'boolean' },
     },
-    my_str_array: { type: 'keyword' },
+    my_str_array: { type: 'array', items: { type: 'keyword' } },
     my_index_signature_prop: {
       count: { type: 'number' },
       avg: { type: 'number' },

diff --git a/src/plugins/home/server/services/sample_data/usage/collector.ts b/src/plugins/home/server/services/sample_data/usage/collector.ts
@@ -38,12 +38,12 @@ export async function makeSampleDataUsageCollector(
     fetch: fetchProvider(index),
     isReady: () => true,
     schema: {
-      installed: { type: 'keyword' },
+      installed: { type: 'array', items: { type: 'keyword' } },
       last_install_date: { type: 'date' },
       last_install_set: { type: 'keyword' },
       last_uninstall_date: { type: 'date' },
       last_uninstall_set: { type: 'keyword' },
-      uninstalled: { type: 'keyword' },
+      uninstalled: { type: 'array', items: { type: 'keyword' } },
     },
   });
 

diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
@@ -29,7 +29,10 @@
     "sample-data": {
       "properties": {
         "installed": {
-          "type": "keyword"
+          "type": "array",
+          "items": {
+            "type": "keyword"
+          }
         },
         "last_install_date": {
           "type": "date"
@@ -44,7 +47,10 @@
           "type": "keyword"
         },
         "uninstalled": {
-          "type": "keyword"
+          "type": "array",
+          "items": {
+            "type": "keyword"
+          }
         }
       }
     },

diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md
@@ -140,6 +140,14 @@ The `AllowedSchemaTypes` is the list of allowed schema types for the usage field
 'keyword', 'text', 'number', 'boolean', 'long', 'date', 'float'
 ```
 
+### Arrays
+
+If any of your properties is an array, the schema definition must follow the convention below:
+
+```
+{ type: 'array', items: {...mySchemaDefinitionOfTheEntriesInTheArray} }
+```
+
 ### Example
 
 ```ts
@@ -152,6 +160,8 @@ export const myCollector = makeUsageCollector<Usage>({
       some_obj: {
         total: 123,
       },
+      some_array: ['value1', 'value2'],
+      some_array_of_obj: [{total: 123}],
     };
   },
   schema: {
@@ -163,6 +173,18 @@ export const myCollector = makeUsageCollector<Usage>({
         type: 'number',
       },
     },
+    some_array: {
+      type: 'array',
+      items: { type: 'keyword' }    
+    },
+    some_array_of_obj: {
+      type: 'array',
+      items: { 
+        total: {
+          type: 'number',
+        },
+      },   
+    },
   },
 });
 ```

diff --git a/src/plugins/usage_collection/server/collector/collector.test.ts b/src/plugins/usage_collection/server/collector/collector.test.ts
@@ -153,7 +153,10 @@ describe('collector', () => {
         isReady: () => false,
         fetch: () => ({ testPass: [{ name: 'a', value: 100 }] }),
         schema: {
-          testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
+          testPass: {
+            type: 'array',
+            items: { name: { type: 'keyword' }, value: { type: 'long' } },
+          },
         },
       });
       expect(collector).toBeDefined();
@@ -166,7 +169,10 @@ describe('collector', () => {
         fetch: () => ({ testPass: [{ name: 'a', value: 100 }], otherProp: 1 }),
         // @ts-expect-error
         schema: {
-          testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
+          testPass: {
+            type: 'array',
+            items: { name: { type: 'keyword' }, value: { type: 'long' } },
+          },
         },
       });
       expect(collector).toBeDefined();
@@ -185,7 +191,10 @@ describe('collector', () => {
         },
         // @ts-expect-error
         schema: {
-          testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
+          testPass: {
+            type: 'array',
+            items: { name: { type: 'keyword' }, value: { type: 'long' } },
+          },
         },
       });
       expect(collector).toBeDefined();
@@ -203,7 +212,10 @@ describe('collector', () => {
           return { otherProp: 1 };
         },
         schema: {
-          testPass: { name: { type: 'keyword' }, value: { type: 'long' } },
+          testPass: {
+            type: 'array',
+            items: { name: { type: 'keyword' }, value: { type: 'long' } },
+          },
           otherProp: { type: 'long' },
         },
       });

diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts
@@ -40,7 +40,7 @@ export type RecursiveMakeSchemaFrom<U> = U extends object
 
 export type MakeSchemaFrom<Base> = {
   [Key in keyof Base]: Base[Key] extends Array<infer U>
-    ? RecursiveMakeSchemaFrom<U>
+    ? { type: 'array'; items: RecursiveMakeSchemaFrom<U> }
     : RecursiveMakeSchemaFrom<Base[Key]>;
 };
 

diff --git a/x-pack/plugins/ingest_manager/server/collectors/register.ts b/x-pack/plugins/ingest_manager/server/collectors/register.ts
@@ -50,9 +50,12 @@ export function registerIngestManagerUsageCollector(
         offline: { type: 'long' },
       },
       packages: {
-        name: { type: 'keyword' },
-        version: { type: 'keyword' },
-        enabled: { type: 'boolean' },
+        type: 'array',
+        items: {
+          name: { type: 'keyword' },
+          version: { type: 'keyword' },
+          enabled: { type: 'boolean' },
+        },
       },
     },
   });