[data views] allow fields that start with an underscore in the field …

…list (elastic#111238) * stop filtering out fields that start with underscore
kibanamachine · Sep 13, 2021 · 83ff4b9 · 83ff4b9
1 parent 0a6dbab
commit 83ff4b9
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 8 deletions.
diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts
@@ -57,9 +57,10 @@ describe('flattenHit', () => {
       fields: {
         date: ['1'],
         zzz: ['z'],
+        _abc: ['a'],
       },
     });
-    const expectedOrder = ['date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
+    const expectedOrder = ['_abc', 'date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
     expect(Object.keys(response)).toEqual(expectedOrder);
     expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
   });

diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts
@@ -66,7 +66,6 @@ function decorateFlattenedWrapper(hit: Record<string, any>, metaFields: Record<s
 
     // unwrap computed fields
     _.forOwn(hit.fields, function (val, key: any) {
-      if (key[0] === '_' && !_.includes(metaFields, key)) return;
       // Flatten an array with 0 or 1 elements to a single value.
       if (Array.isArray(val) && val.length <= 1) {
         flattened[key] = val[0];

diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts
@@ -25,6 +25,7 @@ export interface FieldDescriptor {
   type: string;
   esTypes: string[];
   subType?: FieldSubType;
+  metadata_field?: boolean;
 }
 
 interface FieldSubType {

diff --git a/...gins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js b/...gins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js
@@ -69,6 +69,17 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   });
 
   describe('response order', () => {
+    it('supports fields that start with an underscore', async () => {
+      const fields = ['_field_a', '_field_b'];
+
+      stubDeps({
+        fieldsFromFieldCaps: fields.map((name) => ({ name })),
+      });
+
+      const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
+      expect(fieldNames).toEqual(fields);
+    });
+
     it('always returns fields in alphabetical order', async () => {
       const letters = 'ambcdfjopngihkel'.split('');
       const sortedLetters = sortBy(letters);

diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts
@@ -34,20 +34,21 @@ export async function getFieldCapabilities(
   const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name');
 
   const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName)
-    .filter((name) => !name.startsWith('_'))
+    // not all meta fields are provided, so remove and manually add
+    .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field)
     .concat(metaFields)
-    .reduce<{ names: string[]; hash: Record<string, string> }>(
+    .reduce<{ names: string[]; map: Map<string, string> }>(
       (agg, value) => {
-        // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
-        if (agg.hash[value] != null) {
+        // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data
+        if (agg.map.get(value) != null) {
           return agg;
         } else {
-          agg.hash[value] = value;
+          agg.map.set(value, value);
           agg.names.push(value);
           return agg;
         }
       },
-      { names: [], hash: {} }
+      { names: [], map: new Map<string, string>() }
     )
     .names.map<FieldDescriptor>((name) =>
       defaults({}, fieldsFromFieldCapsByName[name], {
@@ -56,6 +57,7 @@ export async function getFieldCapabilities(
         searchable: false,
         aggregatable: false,
         readFromDocValues: false,
+        metadata_field: metaFields.includes(name),
       })
     )
     .map(mergeOverrides);

diff --git a/...ins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js b/...ins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js
@@ -48,6 +48,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
               'searchable',
               'aggregatable',
               'readFromDocValues',
+              'metadata_field',
             ]);
           });
         }

diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts
@@ -116,6 +116,8 @@ export function readFieldCapsResponse(
             }),
             {}
           ),
+          // @ts-expect-error
+          metadata_field: capsByType[types[0]].metadata_field,
         };
         // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
         agg.array.push(field);
@@ -131,6 +133,8 @@ export function readFieldCapsResponse(
         searchable: isSearchable,
         aggregatable: isAggregatable,
         readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
+        // @ts-expect-error
+        metadata_field: capsByType[types[0]].metadata_field,
       };
       // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
       agg.array.push(field);

diff --git a/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js b/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js
@@ -38,6 +38,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan01',
@@ -46,6 +47,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -54,6 +56,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
             ],
           });
@@ -77,6 +80,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -85,6 +89,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
             ],
           });
@@ -109,6 +114,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -117,20 +123,23 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'meta1',
                 type: 'string',
                 aggregatable: false,
                 searchable: false,
                 readFromDocValues: false,
+                metadata_field: true,
               },
               {
                 name: 'meta2',
                 type: 'string',
                 aggregatable: false,
                 searchable: false,
                 readFromDocValues: false,
+                metadata_field: true,
               },
             ],
           });

diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
@@ -35,6 +35,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'number_conflict',
@@ -43,6 +44,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'string_conflict',
@@ -51,6 +53,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: false,
+                metadata_field: false,
               },
               {
                 name: 'success',
@@ -63,6 +66,7 @@ export default function ({ getService }) {
                   boolean: ['logs-2017.01.02'],
                   keyword: ['logs-2017.01.01'],
                 },
+                metadata_field: false,
               },
             ],
           });

diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
@@ -25,6 +25,7 @@ export default function ({ getService }) {
       aggregatable: true,
       name: 'bar',
       readFromDocValues: true,
+      metadata_field: false,
     },
     {
       type: 'string',
@@ -33,6 +34,7 @@ export default function ({ getService }) {
       aggregatable: false,
       name: 'baz',
       readFromDocValues: false,
+      metadata_field: false,
     },
     {
       type: 'string',
@@ -42,6 +44,7 @@ export default function ({ getService }) {
       name: 'baz.keyword',
       readFromDocValues: true,
       subType: { multi: { parent: 'baz' } },
+      metadata_field: false,
     },
     {
       type: 'number',
@@ -50,6 +53,7 @@ export default function ({ getService }) {
       aggregatable: true,
       name: 'foo',
       readFromDocValues: true,
+      metadata_field: false,
     },
     {
       aggregatable: true,
@@ -63,6 +67,7 @@ export default function ({ getService }) {
         },
       },
       type: 'string',
+      metadata_field: false,
     },
   ];
 
@@ -100,6 +105,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: true,
               type: 'string',
+              metadata_field: true,
             },
             {
               aggregatable: false,
@@ -108,6 +114,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: false,
               type: '_source',
+              metadata_field: true,
             },
             {
               type: 'boolean',
@@ -116,6 +123,7 @@ export default function ({ getService }) {
               aggregatable: true,
               name: 'bar',
               readFromDocValues: true,
+              metadata_field: false,
             },
             {
               aggregatable: false,
@@ -124,6 +132,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: true,
               type: 'string',
+              metadata_field: false,
             },
             {
               type: 'string',
@@ -133,13 +142,15 @@ export default function ({ getService }) {
               name: 'baz.keyword',
               readFromDocValues: true,
               subType: { multi: { parent: 'baz' } },
+              metadata_field: false,
             },
             {
               aggregatable: false,
               name: 'crazy_meta_field',
               readFromDocValues: false,
               searchable: false,
               type: 'string',
+              metadata_field: true,
             },
             {
               type: 'number',
@@ -148,6 +159,7 @@ export default function ({ getService }) {
               aggregatable: true,
               name: 'foo',
               readFromDocValues: true,
+              metadata_field: false,
             },
             {
               aggregatable: true,
@@ -161,6 +173,7 @@ export default function ({ getService }) {
                 },
               },
               type: 'string',
+              metadata_field: false,
             },
           ],
         })