Merge branch 'master' into create-saved-objs-svc

elastic · Feb 23, 2021 · 89a607e · 89a607e
2 parents 8860be5 + 5356866
commit 89a607e
Show file tree

Hide file tree

Showing 225 changed files with 2,037 additions and 1,367 deletions.
diff --git a/.ci/Jenkinsfile_security_cypress b/.ci/Jenkinsfile_security_cypress
@@ -18,7 +18,8 @@ kibanaPipeline(timeoutMinutes: 180) {
         workers.ci(name: job, size: 'l', ramDisk: true) {
           kibanaPipeline.bash('test/scripts/jenkins_xpack_build_kibana.sh', 'Build Default Distributable')
           kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress_chrome.sh')()
-          kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress_firefox.sh')()
+          // Temporarily disabled to figure out test flake
+          // kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress_firefox.sh')()
         }
       }
     }

diff --git a/.node-version b/.node-version
@@ -1 +1 @@
-14.15.4
+14.16.0
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-14.15.4
+14.16.0
diff --git a/package.json b/package.json
@@ -89,7 +89,7 @@
     "**/typescript": "4.1.3"
   },
   "engines": {
-    "node": "14.15.4",
+    "node": "14.16.0",
     "yarn": "^1.21.1"
   },
   "dependencies": {

diff --git a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts
@@ -25,7 +25,7 @@ export async function emptyKibanaIndexAction({
   const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   await cleanKibanaIndices({ client, stats, log, kibanaPluginIds });
-  await migrateKibanaIndex({ client, kbnClient });
+  await migrateKibanaIndex(kbnClient);
   stats.createdIndex('.kibana');
   return stats.toJSON();
 }
diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts
@@ -12,8 +12,8 @@ import { Readable } from 'stream';
 import { ToolingLog } from '@kbn/dev-utils';
 import { KbnClient } from '@kbn/test';
 import { Client } from '@elastic/elasticsearch';
-
 import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
+import { ES_CLIENT_HEADERS } from '../client_headers';
 
 import {
   isGzip,
@@ -91,14 +91,19 @@ export async function loadAction({
     }
   }
 
-  await client.indices.refresh({
-    index: '_all',
-    allow_no_indices: true,
-  });
+  await client.indices.refresh(
+    {
+      index: '_all',
+      allow_no_indices: true,
+    },
+    {
+      headers: ES_CLIENT_HEADERS,
+    }
+  );
 
   // If we affected the Kibana index, we need to ensure it's migrated...
   if (Object.keys(result).some((k) => k.startsWith('.kibana'))) {
-    await migrateKibanaIndex({ client, kbnClient });
+    await migrateKibanaIndex(kbnClient);
     log.debug('[%s] Migrated Kibana index after loading Kibana data', name);
 
     if (kibanaPluginIds.includes('spaces')) {

diff --git a/packages/kbn-es-archiver/src/client_headers.ts b/packages/kbn-es-archiver/src/client_headers.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const ES_CLIENT_HEADERS = {
+  'x-elastic-product-origin': 'kibana',
+};
diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.test.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.test.ts
@@ -107,6 +107,11 @@ it('transforms each input index to a stream of docs using scrollSearch helper',
             "scroll": "1m",
             "size": 1000,
           },
+          Object {
+            "headers": Object {
+              "x-elastic-product-origin": "kibana",
+            },
+          },
         ],
         Array [
           Object {
@@ -119,6 +124,11 @@ it('transforms each input index to a stream of docs using scrollSearch helper',
             "scroll": "1m",
             "size": 1000,
           },
+          Object {
+            "headers": Object {
+              "x-elastic-product-origin": "kibana",
+            },
+          },
         ],
       ],
       "results": Array [

diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts
@@ -10,6 +10,7 @@ import { Transform } from 'stream';
 import { Client } from '@elastic/elasticsearch';
 import { Stats } from '../stats';
 import { Progress } from '../progress';
+import { ES_CLIENT_HEADERS } from '../../client_headers';
 
 const SCROLL_SIZE = 1000;
 const SCROLL_TIMEOUT = '1m';
@@ -30,16 +31,21 @@ export function createGenerateDocRecordsStream({
     readableObjectMode: true,
     async transform(index, enc, callback) {
       try {
-        const interator = client.helpers.scrollSearch({
-          index,
-          scroll: SCROLL_TIMEOUT,
-          size: SCROLL_SIZE,
-          _source: 'true',
-          body: {
-            query,
+        const interator = client.helpers.scrollSearch(
+          {
+            index,
+            scroll: SCROLL_TIMEOUT,
+            size: SCROLL_SIZE,
+            _source: 'true',
+            body: {
+              query,
+            },
+            rest_total_hits_as_int: true,
           },
-          rest_total_hits_as_int: true,
-        });
+          {
+            headers: ES_CLIENT_HEADERS,
+          }
+        );
 
         let remainingHits: number | null = null;
 

diff --git a/packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.test.ts b/packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.test.ts
@@ -152,6 +152,11 @@ it('indexes documents using the bulk client helper', async () => {
             "onDrop": [Function],
             "retries": 5,
           },
+          Object {
+            "headers": Object {
+              "x-elastic-product-origin": "kibana",
+            },
+          },
         ],
         Array [
           Object {
@@ -170,6 +175,11 @@ it('indexes documents using the bulk client helper', async () => {
             "onDrop": [Function],
             "retries": 5,
           },
+          Object {
+            "headers": Object {
+              "x-elastic-product-origin": "kibana",
+            },
+          },
         ],
       ],
       "results": Array [

diff --git a/packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts
@@ -11,6 +11,7 @@ import AggregateError from 'aggregate-error';
 import { Writable } from 'stream';
 import { Stats } from '../stats';
 import { Progress } from '../progress';
+import { ES_CLIENT_HEADERS } from '../../client_headers';
 
 export function createIndexDocRecordsStream(
   client: Client,
@@ -23,27 +24,32 @@ export function createIndexDocRecordsStream(
     const ops = new WeakMap<any, any>();
     const errors: string[] = [];
 
-    await client.helpers.bulk({
-      retries: 5,
-      datasource: docs.map((doc) => {
-        const body = doc.source;
-        ops.set(body, {
-          [operation]: {
-            _index: doc.index,
-            _id: doc.id,
-          },
-        });
-        return body;
-      }),
-      onDocument(doc) {
-        return ops.get(doc);
+    await client.helpers.bulk(
+      {
+        retries: 5,
+        datasource: docs.map((doc) => {
+          const body = doc.source;
+          ops.set(body, {
+            [operation]: {
+              _index: doc.index,
+              _id: doc.id,
+            },
+          });
+          return body;
+        }),
+        onDocument(doc) {
+          return ops.get(doc);
+        },
+        onDrop(dropped) {
+          const dj = JSON.stringify(dropped.document);
+          const ej = JSON.stringify(dropped.error);
+          errors.push(`Bulk doc failure [operation=${operation}]:\n  doc: ${dj}\n  error: ${ej}`);
+        },
       },
-      onDrop(dropped) {
-        const dj = JSON.stringify(dropped.document);
-        const ej = JSON.stringify(dropped.error);
-        errors.push(`Bulk doc failure [operation=${operation}]:\n  doc: ${dj}\n  error: ${ej}`);
-      },
-    });
+      {
+        headers: ES_CLIENT_HEADERS,
+      }
+    );
 
     if (errors.length) {
       throw new AggregateError(errors);

diff --git a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.ts
@@ -65,6 +65,9 @@ describe('esArchiver: createCreateIndexStream()', () => {
               ],
             },
             Object {
+              "headers": Object {
+                "x-elastic-product-origin": "kibana",
+              },
               "ignore": Array [
                 404,
               ],
@@ -81,6 +84,11 @@ describe('esArchiver: createCreateIndexStream()', () => {
                 "actual-index",
               ],
             },
+            Object {
+              "headers": Object {
+                "x-elastic-product-origin": "kibana",
+              },
+            },
           ],
         ]
       `);

diff --git a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts
@@ -15,6 +15,7 @@ import { ToolingLog } from '@kbn/dev-utils';
 import { Stats } from '../stats';
 import { deleteKibanaIndices } from './kibana_index';
 import { deleteIndex } from './delete_index';
+import { ES_CLIENT_HEADERS } from '../../client_headers';
 
 interface DocRecord {
   value: {
@@ -63,15 +64,20 @@ export function createCreateIndexStream({
           kibanaIndexAlreadyDeleted = true;
         }
 
-        await client.indices.create({
-          method: 'PUT',
-          index,
-          body: {
-            settings,
-            mappings,
-            aliases,
+        await client.indices.create(
+          {
+            method: 'PUT',
+            index,
+            body: {
+              settings,
+              mappings,
+              aliases,
+            },
           },
-        });
+          {
+            headers: ES_CLIENT_HEADERS,
+          }
+        );
 
         stats.createdIndex(index, { settings });
       } catch (err) {

diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index.ts
@@ -9,6 +9,7 @@
 import { Client } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 import { Stats } from '../stats';
+import { ES_CLIENT_HEADERS } from '../../client_headers';
 
 // see https://github.com/elastic/elasticsearch/blob/99f88f15c5febbca2d13b5b5fda27b844153bf1a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java#L313-L319
 const PENDING_SNAPSHOT_STATUSES = ['INIT', 'STARTED', 'WAITING'];
@@ -30,6 +31,7 @@ export async function deleteIndex(options: {
       },
       {
         ignore: [404],
+        headers: ES_CLIENT_HEADERS,
       }
     );
 
@@ -38,7 +40,12 @@ export async function deleteIndex(options: {
 
   try {
     const indicesToDelete = await getIndicesToDelete();
-    await client.indices.delete({ index: indicesToDelete });
+    await client.indices.delete(
+      { index: indicesToDelete },
+      {
+        headers: ES_CLIENT_HEADERS,
+      }
+    );
     for (const index of indices) {
       stats.deletedIndex(index);
     }
@@ -86,10 +93,15 @@ export async function waitForSnapshotCompletion(
       body: {
         snapshots: [status],
       },
-    } = await client.snapshot.status({
-      repository,
-      snapshot,
-    });
+    } = await client.snapshot.status(
+      {
+        repository,
+        snapshot,
+      },
+      {
+        headers: ES_CLIENT_HEADERS,
+      }
+    );
 
     log.debug(`Snapshot ${repository}/${snapshot} is ${status.state}`);
     return PENDING_SNAPSHOT_STATUSES.includes(status.state);
@@ -98,15 +110,21 @@ export async function waitForSnapshotCompletion(
   const getInProgressSnapshots = async (repository: string) => {
     const {
       body: { snapshots: inProgressSnapshots },
-    } = await client.snapshot.get({
-      repository,
-      snapshot: '_current',
-    });
+    } = await client.snapshot.get(
+      {
+        repository,
+        snapshot: '_current',
+      },
+      {
+        headers: ES_CLIENT_HEADERS,
+      }
+    );
 
     return inProgressSnapshots;
   };
 
-  for (const repository of Object.keys(await client.snapshot.getRepository({} as any))) {
+  const { body: repositoryMap } = await client.snapshot.getRepository({} as any);
+  for (const repository of Object.keys(repositoryMap)) {
     const allInProgress = await getInProgressSnapshots(repository);
     const found = allInProgress.find((s: any) => s.indices.includes(index));