From fa8adf199612fe24ff446195e9570fd5fe5336b4 Mon Sep 17 00:00:00 2001
From: Tim Sullivan <tsullivan@users.noreply.github.com>
Date: Tue, 9 Feb 2021 14:32:45 -0700
Subject: [PATCH] [Tabify] Add meta option to include top-level underscored
 field values (#90535) (#90814)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../__snapshots__/tabify_docs.test.ts.snap    | 284 ++++++++++++++++++
 .../common/search/tabify/tabify_docs.test.ts  |   9 +
 .../data/common/search/tabify/tabify_docs.ts  |  17 +-
 3 files changed, 305 insertions(+), 5 deletions(-)

diff --git a/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap b/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap
index 6cc191a67633c..22276335a0599 100644
--- a/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap
+++ b/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap
@@ -1,5 +1,113 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`tabifyDocs combines meta fields if meta option is set 1`] = `
+Object {
+  "columns": Array [
+    Object {
+      "id": "fieldTest",
+      "meta": Object {
+        "field": "fieldTest",
+        "index": "test-index",
+        "params": Object {
+          "id": "number",
+        },
+        "type": "number",
+      },
+      "name": "fieldTest",
+    },
+    Object {
+      "id": "invalidMapping",
+      "meta": Object {
+        "field": "invalidMapping",
+        "index": "test-index",
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "invalidMapping",
+    },
+    Object {
+      "id": "nested",
+      "meta": Object {
+        "field": "nested",
+        "index": "test-index",
+        "params": undefined,
+        "type": "object",
+      },
+      "name": "nested",
+    },
+    Object {
+      "id": "sourceTest",
+      "meta": Object {
+        "field": "sourceTest",
+        "index": "test-index",
+        "params": Object {
+          "id": "number",
+        },
+        "type": "number",
+      },
+      "name": "sourceTest",
+    },
+    Object {
+      "id": "_id",
+      "meta": Object {
+        "field": "_id",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_id",
+    },
+    Object {
+      "id": "_index",
+      "meta": Object {
+        "field": "_index",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_index",
+    },
+    Object {
+      "id": "_score",
+      "meta": Object {
+        "field": "_score",
+        "index": "test-index",
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "_score",
+    },
+    Object {
+      "id": "_type",
+      "meta": Object {
+        "field": "_type",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_type",
+    },
+  ],
+  "rows": Array [
+    Object {
+      "_id": "hit-id-value",
+      "_index": "hit-index-value",
+      "_score": 77,
+      "_type": "hit-type-value",
+      "fieldTest": 123,
+      "invalidMapping": 345,
+      "nested": Array [
+        Object {
+          "field": 123,
+        },
+      ],
+      "sourceTest": 123,
+    },
+  ],
+  "type": "datatable",
+}
+`;
+
 exports[`tabifyDocs converts fields by default 1`] = `
 Object {
   "columns": Array [
@@ -47,9 +155,53 @@ Object {
       },
       "name": "sourceTest",
     },
+    Object {
+      "id": "_id",
+      "meta": Object {
+        "field": "_id",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_id",
+    },
+    Object {
+      "id": "_index",
+      "meta": Object {
+        "field": "_index",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_index",
+    },
+    Object {
+      "id": "_score",
+      "meta": Object {
+        "field": "_score",
+        "index": "test-index",
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "_score",
+    },
+    Object {
+      "id": "_type",
+      "meta": Object {
+        "field": "_type",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_type",
+    },
   ],
   "rows": Array [
     Object {
+      "_id": "hit-id-value",
+      "_index": "hit-index-value",
+      "_score": 77,
+      "_type": "hit-type-value",
       "fieldTest": 123,
       "invalidMapping": 345,
       "nested": Array [
@@ -111,9 +263,53 @@ Object {
       },
       "name": "sourceTest",
     },
+    Object {
+      "id": "_id",
+      "meta": Object {
+        "field": "_id",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_id",
+    },
+    Object {
+      "id": "_index",
+      "meta": Object {
+        "field": "_index",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_index",
+    },
+    Object {
+      "id": "_score",
+      "meta": Object {
+        "field": "_score",
+        "index": "test-index",
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "_score",
+    },
+    Object {
+      "id": "_type",
+      "meta": Object {
+        "field": "_type",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_type",
+    },
   ],
   "rows": Array [
     Object {
+      "_id": "hit-id-value",
+      "_index": "hit-index-value",
+      "_score": 77,
+      "_type": "hit-type-value",
       "fieldTest": 123,
       "invalidMapping": 345,
       "nested": Array [
@@ -175,9 +371,53 @@ Object {
       },
       "name": "sourceTest",
     },
+    Object {
+      "id": "_id",
+      "meta": Object {
+        "field": "_id",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_id",
+    },
+    Object {
+      "id": "_index",
+      "meta": Object {
+        "field": "_index",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_index",
+    },
+    Object {
+      "id": "_score",
+      "meta": Object {
+        "field": "_score",
+        "index": "test-index",
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "_score",
+    },
+    Object {
+      "id": "_type",
+      "meta": Object {
+        "field": "_type",
+        "index": "test-index",
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_type",
+    },
   ],
   "rows": Array [
     Object {
+      "_id": "hit-id-value",
+      "_index": "hit-index-value",
+      "_score": 77,
+      "_type": "hit-type-value",
       "fieldTest": 123,
       "invalidMapping": 345,
       "nested": Array [
@@ -235,9 +475,53 @@ Object {
       },
       "name": "sourceTest",
     },
+    Object {
+      "id": "_id",
+      "meta": Object {
+        "field": "_id",
+        "index": undefined,
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_id",
+    },
+    Object {
+      "id": "_index",
+      "meta": Object {
+        "field": "_index",
+        "index": undefined,
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_index",
+    },
+    Object {
+      "id": "_score",
+      "meta": Object {
+        "field": "_score",
+        "index": undefined,
+        "params": undefined,
+        "type": "number",
+      },
+      "name": "_score",
+    },
+    Object {
+      "id": "_type",
+      "meta": Object {
+        "field": "_type",
+        "index": undefined,
+        "params": undefined,
+        "type": "string",
+      },
+      "name": "_type",
+    },
   ],
   "rows": Array [
     Object {
+      "_id": "hit-id-value",
+      "_index": "hit-index-value",
+      "_score": 77,
+      "_type": "hit-type-value",
       "fieldTest": 123,
       "invalidMapping": 345,
       "nested": Array [
diff --git a/src/plugins/data/common/search/tabify/tabify_docs.test.ts b/src/plugins/data/common/search/tabify/tabify_docs.test.ts
index c81e39f4c156a..52e12aeee1ae6 100644
--- a/src/plugins/data/common/search/tabify/tabify_docs.test.ts
+++ b/src/plugins/data/common/search/tabify/tabify_docs.test.ts
@@ -37,6 +37,10 @@ describe('tabifyDocs', () => {
     hits: {
       hits: [
         {
+          _id: 'hit-id-value',
+          _index: 'hit-index-value',
+          _type: 'hit-type-value',
+          _score: 77,
           _source: { sourceTest: 123 },
           fields: { fieldTest: 123, invalidMapping: 345, nested: [{ field: 123 }] },
         },
@@ -59,6 +63,11 @@ describe('tabifyDocs', () => {
     expect(table).toMatchSnapshot();
   });
 
+  it('combines meta fields if meta option is set', () => {
+    const table = tabifyDocs(response, index, { meta: true });
+    expect(table).toMatchSnapshot();
+  });
+
   it('works without provided index pattern', () => {
     const table = tabifyDocs(response);
     expect(table).toMatchSnapshot();
diff --git a/src/plugins/data/common/search/tabify/tabify_docs.ts b/src/plugins/data/common/search/tabify/tabify_docs.ts
index eaf43d9fd6ff6..b4806283e63f2 100644
--- a/src/plugins/data/common/search/tabify/tabify_docs.ts
+++ b/src/plugins/data/common/search/tabify/tabify_docs.ts
@@ -11,6 +11,12 @@ import { isPlainObject } from 'lodash';
 import { IndexPattern } from '../../index_patterns/index_patterns';
 import { Datatable, DatatableColumn, DatatableColumnType } from '../../../../expressions/common';
 
+export interface TabifyDocsOptions {
+  shallow?: boolean;
+  source?: boolean;
+  meta?: boolean;
+}
+
 export function flattenHit(
   hit: SearchResponse<unknown>['hits']['hits'][0],
   indexPattern?: IndexPattern,
@@ -56,12 +62,13 @@ export function flattenHit(
   if (params?.source !== false && hit._source) {
     flatten(hit._source as Record<string, any>);
   }
-  return flat;
-}
+  if (params?.meta !== false) {
+    // combine the fields that Discover allows to add as columns
+    const { _id, _index, _type, _score } = hit;
+    flatten({ _id, _index, _score, _type });
+  }
 
-export interface TabifyDocsOptions {
-  shallow?: boolean;
-  source?: boolean;
+  return flat;
 }
 
 export const tabifyDocs = (