[Discover] Fix filtering out custom meta fields of Elasticsearch plugins enhanced documents

src/plugins/data/common/search/tabify/tabify_docs.test.ts

@@ -63,6 +63,8 @@ describe('tabify_docs', () => {
           _source: {
             name: 'first',
           },
+          _routing: 'test',
+          _score: 1,
           fields: {
             date: ['1'],
             zzz: ['z'],

src/plugins/data/common/search/tabify/tabify_docs.ts

@@ -25,25 +25,12 @@ type ValidMetaFieldNames = keyof Pick<
   | '_source'
   | '_version'
 >;
-const VALID_META_FIELD_NAMES: ValidMetaFieldNames[] = [
-  '_id',
-  '_ignored',
-  '_index',
-  '_node',
-  '_primary_term',
-  '_routing',
-  '_score',
-  '_seq_no',
-  '_shard',
-  '_source',
-  '_version',
-];
 
 function isValidMetaFieldName(field: string): field is ValidMetaFieldNames {
   // Since the array above is more narrowly typed than string[], we cannot use
   // string to find a value in here. We manually cast it to wider string[] type
   // so we're able to use `includes` on it.
-  return (VALID_META_FIELD_NAMES as string[]).includes(field);
+  return field !== '_source' && field !== '_type' && field.at(0) === '_';
 }
 
 interface TabifyDocsOptions {
@@ -137,13 +124,14 @@ export function flattenHit(hit: Hit, indexPattern?: DataView, params?: TabifyDoc
     });
   }
 
-  // Merge all valid meta fields into the flattened object
-  // expect for _source (in case that was specified as a meta field)
+  // Merge all meta fields into the flattened object
   indexPattern?.metaFields?.forEach((metaFieldName) => {
-    if (!isValidMetaFieldName(metaFieldName) || metaFieldName === '_source') {
+    if (!isValidMetaFieldName(metaFieldName)) {
       return;
     }
-    flat[metaFieldName] = hit[metaFieldName];
+    if (hit[metaFieldName] !== undefined && !flat[metaFieldName]) {
+      flat[metaFieldName] = hit[metaFieldName];
+    }
   });
 
   // Use a proxy to make sure that keys are always returned in a specific order,

src/plugins/discover/public/utils/format_hit.test.ts

@@ -49,7 +49,6 @@ describe('formatHit', () => {
       ['message', 'formatted:foobar'],
       ['object.value', 'formatted:42,13'],
       ['_index', 'formatted:logs'],
-      ['_score', undefined],
     ]);
   });
 
@@ -74,7 +73,6 @@ describe('formatHit', () => {
       'extension',
       'object.value',
       '_index',
-      '_score',
     ]);
   });
 
@@ -89,7 +87,7 @@ describe('formatHit', () => {
     expect(formatted).toEqual([
       ['extension', 'formatted:png'],
       ['message', 'formatted:foobar'],
-      ['and 3 more fields', ''],
+      ['and 2 more fields', ''],
     ]);
   });
 
@@ -105,7 +103,6 @@ describe('formatHit', () => {
       ['message', 'formatted:foobar'],
       ['object.value', 'formatted:42,13'],
       ['_index', 'formatted:logs'],
-      ['_score', undefined],
     ]);
   });
 
@@ -120,7 +117,6 @@ describe('formatHit', () => {
     expect(formatted).toEqual([
       ['bytesDisplayName', 'formatted:123'],
       ['_index', 'formatted:logs'],
-      ['_score', undefined],
     ]);
   });
 });

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts

@@ -453,6 +453,7 @@ describe('fields from job.searchSource.getFields() (7.12 generated)', () => {
               {
                 _id: 'my-cool-id',
                 _index: 'my-cool-index',
+                _score: 1,
                 _version: 4,
                 fields: {
                   date: ['2020-12-31T00:14:28.000Z'],