Merge remote-tracking branch 'upstream/master' into move-kbn-config-i…

…nto-bazel

api_docs/usage_collection.mdx

@@ -22,13 +22,13 @@ import usageCollectionObj from './usage_collection.json';
 ### Functions
 <DocDefinitionList data={usageCollectionObj.client.functions}/>
 
-### Enums
-<DocDefinitionList data={usageCollectionObj.client.enums}/>
+### Interfaces
+<DocDefinitionList data={usageCollectionObj.client.interfaces}/>
 
 ## Server
 
-### Classes
-<DocDefinitionList data={usageCollectionObj.server.classes}/>
+### Setup
+<DocDefinitionList data={[usageCollectionObj.server.setup]}/>
 
 ### Interfaces
 <DocDefinitionList data={usageCollectionObj.server.interfaces}/>

docs/developer/getting-started/building-kibana.asciidoc

@@ -18,12 +18,12 @@ yarn build --help
 [discrete]
 === Building OS packages
 
-Packages are built using fpm, dpkg, and rpm.  Package building has only been tested on Linux and is not supported on any other platform.
-
+Packages are built using fpm, dpkg, and rpm, and Docker.  Package building has only been tested on Linux and is not supported on any other platform.
+Docker installation instructions can be found at https://docs.docker.com/engine/install/[Install Docker Engine].
 
 [source,bash]
 ----
-apt-get install ruby-dev rpm
+apt-get install ruby ruby-dev rpm dpkg build-essential
 gem install fpm -v 1.5.0
 yarn build --skip-archives
 ----

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -104,6 +104,9 @@ readonly links: {
             readonly painlessWalkthrough: string;
             readonly luceneExpressions: string;
         };
+        readonly search: {
+            readonly sessions: string;
+        };
         readonly indexPatterns: {
             readonly introduction: string;
             readonly fieldFormattersNumber: string;

docs/user/alerting/alerting-troubleshooting.asciidoc

@@ -69,3 +69,167 @@ Configuration options are available to specialize connections to TLS servers,
 including ignoring server certificate validation, and providing certificate
 authority data to verify servers using custom certificates.  For more details, 
 see <<action-settings,Action settings>>.
+
+[float]
+[[rules-long-execution-time]]
+=== Identify long-running rules
+
+The following query can help you identify rules that are taking a long time to execute and might impact the overall health of your deployment.
+
+[IMPORTANT]
+==============================================
+By default, only users with a `superuser` role can query the {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index.
+==============================================
+
+Query for a list of rule ids, bucketed by their execution times:
+
+[source,console]
+--------------------------------------------------
+GET /.kibana-event-log*/_search
+{
+  "size": 0,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "range": {
+            "@timestamp": {
+              "gte": "now-1d", <1>
+              "lte": "now"
+            }
+          }
+        },
+        {
+          "term": {
+            "event.action": {
+              "value": "execute"
+            }
+          }
+        },
+        {
+          "term": {
+            "event.provider": {
+              "value": "alerting" <2>
+            }
+          }
+        }
+      ]
+    }
+  },
+  "runtime_mappings": { <3>
+    "event.duration_in_seconds": {
+      "type": "double",
+      "script": {
+        "source": "emit(doc['event.duration'].value / 1E9)"
+      }
+    }
+  },
+  "aggs": {
+    "ruleIdsByExecutionDuration": {
+      "histogram": {
+        "field": "event.duration_in_seconds",
+        "min_doc_count": 1,
+        "interval": 1 <4>
+      },
+      "aggs": {
+        "ruleId": {
+          "nested": {
+            "path": "kibana.saved_objects"
+          },
+          "aggs": {
+            "ruleId": {
+              "terms": {
+                "field": "kibana.saved_objects.id",
+                "size": 10 <5>
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TEST
+
+<1> This queries for rules executed in the last day. Update the values of `lte` and `gte` to query over a different time range.
+<2> Use `event.provider: actions` to query for long-running action executions.
+<3> Execution durations are stored as nanoseconds. This adds a runtime field to convert that duration into seconds.
+<4> This interval buckets the event.duration_in_seconds runtime field into 1 second intervals. Update this value to change the granularity of the buckets. If you are unable to use runtime fields, make sure this aggregation targets `event.duration` and use nanoseconds for the interval.
+<5> This retrieves the top 10 rule ids for this duration interval. Update this value to retrieve more rule ids.
+
+This query returns the following:
+
+[source,json]
+--------------------------------------------------
+{
+  "took" : 322,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 326,
+      "relation" : "eq"
+    },
+    "max_score" : null,
+    "hits" : [ ]
+  },
+  "aggregations" : {
+    "ruleIdsByExecutionDuration" : {
+      "buckets" : [
+        {
+          "key" : 0.0, <1>
+          "doc_count" : 320,
+          "ruleId" : {
+            "doc_count" : 320,
+            "ruleId" : {
+              "doc_count_error_upper_bound" : 0,
+              "sum_other_doc_count" : 0,
+              "buckets" : [
+                {
+                  "key" : "1923ada0-a8f3-11eb-a04b-13d723cdfdc5",
+                  "doc_count" : 140
+                },
+                {
+                  "key" : "15415ecf-cdb0-4fef-950a-f824bd277fe4",
+                  "doc_count" : 130
+                },
+                {
+                  "key" : "dceeb5d0-6b41-11eb-802b-85b0c1bc8ba2",
+                  "doc_count" : 50
+                }
+              ]
+            }
+          }
+        },
+        {
+          "key" : 30.0, <2>
+          "doc_count" : 6,
+          "ruleId" : {
+            "doc_count" : 6,
+            "ruleId" : {
+              "doc_count_error_upper_bound" : 0,
+              "sum_other_doc_count" : 0,
+              "buckets" : [
+                {
+                  "key" : "41893910-6bca-11eb-9e0d-85d233e3ee35",
+                  "doc_count" : 6
+                }
+              ]
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+<1> Most rule execution durations fall within the first bucket (0 - 1 seconds).
+<2> A single rule with id `41893910-6bca-11eb-9e0d-85d233e3ee35` took between 30 and 31 seconds to execute.
+
+Use the <<get-rule-api,Get Rule API>> to retrieve additional information about rules that take a long time to execute.

package.json

@@ -88,7 +88,8 @@
     "**/react-syntax-highlighter/**/highlight.js": "^10.4.1",
     "**/request": "^2.88.2",
     "**/trim": "0.0.3",
-    "**/typescript": "4.1.3"
+    "**/typescript": "4.1.3",
+    "**/underscore": "^1.13.1"
   },
   "engines": {
     "node": "14.16.1",

src/core/public/doc_links/doc_links_service.ts

@@ -199,6 +199,9 @@ export class DocLinksService {
           percolate: `${ELASTICSEARCH_DOCS}query-dsl-percolate-query.html`,
           queryDsl: `${ELASTICSEARCH_DOCS}query-dsl.html`,
         },
+        search: {
+          sessions: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/search-sessions.html`,
+        },
         date: {
           dateMath: `${ELASTICSEARCH_DOCS}common-options.html#date-math`,
           dateMathIndexNames: `${ELASTICSEARCH_DOCS}date-math-index-names.html`,
@@ -501,6 +504,9 @@ export interface DocLinksStart {
       readonly painlessWalkthrough: string;
       readonly luceneExpressions: string;
     };
+    readonly search: {
+      readonly sessions: string;
+    };
     readonly indexPatterns: {
       readonly introduction: string;
       readonly fieldFormattersNumber: string;

src/core/public/http/types.ts

@@ -339,7 +339,7 @@ export interface IHttpFetchError extends Error {
    * @deprecated Provided for legacy compatibility. Prefer the `response` property instead.
    */
   readonly res?: Response;
-  readonly body?: any;
+  readonly body?: any; // TODO: this should be unknown
 }
 
 /** @public */

src/core/public/public.api.md

@@ -589,6 +589,9 @@ export interface DocLinksStart {
             readonly painlessWalkthrough: string;
             readonly luceneExpressions: string;
         };
+        readonly search: {
+            readonly sessions: string;
+        };
         readonly indexPatterns: {
             readonly introduction: string;
             readonly fieldFormattersNumber: string;

src/core/server/saved_objects/migrations/core/document_migrator.ts

@@ -169,7 +169,7 @@ export class DocumentMigrator implements VersionedTransformer {
   }
 
   /**
-   * Gets the latest version of each migratable property.
+   * Gets the latest version of each migrate-able property.
    *
    * @readonly
    * @type {SavedObjectsMigrationVersion}

src/core/server/saved_objects/migrations/core/elastic_index.ts

@@ -30,7 +30,7 @@ export interface FullIndexInfo {
 
 // When migrating from the outdated index we use a read query which excludes
 // saved objects which are no longer used. These saved objects will still be
-// kept in the outdated index for backup purposes, but won't be availble in
+// kept in the outdated index for backup purposes, but won't be available in
 // the upgraded index.
 export const excludeUnusedTypesQuery: estypes.QueryContainer = {
   bool: {

src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts

@@ -344,13 +344,13 @@ const mockV2MigrationOptions = () => {
 
   options.client.openPointInTime = jest
     .fn()
-    .mockImplementationOnce(() =>
+    .mockImplementation(() =>
       elasticsearchClientMock.createSuccessTransportRequestPromise({ id: 'pit_id' })
     );
 
   options.client.closePointInTime = jest
     .fn()
-    .mockImplementationOnce(() =>
+    .mockImplementation(() =>
       elasticsearchClientMock.createSuccessTransportRequestPromise({ succeeded: true })
     );
 

src/core/server/saved_objects/migrationsv2/actions/index.test.ts

@@ -18,7 +18,7 @@ describe('actions', () => {
     jest.clearAllMocks();
   });
 
-  // Create a mock client that rejects all methods with a 503 statuscode
+  // Create a mock client that rejects all methods with a 503 status code
   // response.
   const retryableError = new EsErrors.ResponseError(
     elasticsearchClientMock.createApiResponse({
@@ -92,7 +92,7 @@ describe('actions', () => {
 
   describe('readWithPit', () => {
     it('calls catchRetryableEsClientErrors when the promise rejects', async () => {
-      const task = Actions.readWithPit(client, 'pitId', Option.none, 10_000);
+      const task = Actions.readWithPit(client, 'pitId', { match_all: {} }, 10_000);
       try {
         await task();
       } catch (e) {
@@ -134,7 +134,7 @@ describe('actions', () => {
         'my_target_index',
         Option.none,
         false,
-        Option.none
+        {}
       );
       try {
         await task();
@@ -263,4 +263,17 @@ describe('actions', () => {
       expect(catchRetryableEsClientErrors).toHaveBeenCalledWith(retryableError);
     });
   });
+
+  describe('refreshIndex', () => {
+    it('calls catchRetryableEsClientErrors when the promise rejects', async () => {
+      const task = Actions.refreshIndex(client, 'target_index');
+      try {
+        await task();
+      } catch (e) {
+        /** ignore */
+      }
+
+      expect(catchRetryableEsClientErrors).toHaveBeenCalledWith(retryableError);
+    });
+  });
 });