Merge branch 'master' into snapshots_sorting

.buildkite/pipelines/performance/nightly.yml

@@ -0,0 +1,35 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('ITERATION_COUNT_ENV') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests"
+    command: .buildkite/scripts/steps/functional/performance.sh
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    concurrency: 50
+    concurrency_group: 'performance-test-group'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+

.buildkite/scripts/steps/functional/performance.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -uo pipefail
+
+if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
+  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+else
+  ITERATION_COUNT=$ITERATION_COUNT_ENV
+fi
+
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - command: .buildkite/scripts/steps/functional/performance_sub.sh
+    parallelism: "$ITERATION_COUNT"
+EOF
+
+
+

.buildkite/scripts/steps/functional/performance_sub.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+cd "$XPACK_DIR"
+
+echo --- Run Performance Tests
+checks-reporter-with-killswitch "Run Performance Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config test/performance/config.ts;

.eslintrc.js

@@ -156,6 +156,78 @@ const DEV_PATTERNS = [
   'x-pack/plugins/*/server/scripts/**/*',
 ];
 
+/** Restricted imports with suggested alternatives */
+const RESTRICTED_IMPORTS = [
+  {
+    name: 'lodash',
+    importNames: ['set', 'setWith'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.setwith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['set', 'setWith', 'assoc', 'assocPath'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assoc',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assocPath',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash.template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'react-use',
+    message: 'Please use react-use/lib/{method} instead.',
+  },
+];
+
 module.exports = {
   root: true,
 
@@ -668,81 +740,7 @@ module.exports = {
         'no-restricted-imports': [
           2,
           {
-            paths: [
-              {
-                name: 'lodash',
-                importNames: ['set', 'setWith'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.setwith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['set', 'setWith', 'assoc', 'assocPath'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assoc',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assocPath',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash.template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'react-use',
-                message: 'Please use react-use/lib/{method} instead.',
-              },
-            ],
+            paths: RESTRICTED_IMPORTS,
           },
         ],
         'no-restricted-modules': [
@@ -835,6 +833,23 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ['**/common/**/*.{js,mjs,ts,tsx}', '**/public/**/*.{js,mjs,ts,tsx}'],
+      rules: {
+        'no-restricted-imports': [
+          2,
+          {
+            paths: [
+              ...RESTRICTED_IMPORTS,
+              {
+                name: 'semver',
+                message: 'Please use "semver/*/{function}" instead',
+              },
+            ],
+          },
+        ],
+      },
+    },
 
     /**
      * APM and Observability overrides

docs/management/connectors/action-types/pagerduty.asciidoc

@@ -68,7 +68,7 @@ PagerDuty actions have the following properties.
 
 Severity::      The perceived severity of on the affected system. This can be one of `Critical`, `Error`, `Warning` or `Info`(default).
 Event action::  One of `Trigger` (default), `Resolve`, or `Acknowledge`. See https://v2.developer.pagerduty.com/docs/events-api-v2#event-action[event action] for more details.
-Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<alert ID>:<alert instance ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
+Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<rule ID>:<alert ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
 Timestamp::     An *optional* https://v2.developer.pagerduty.com/v2/docs/types#datetime[ISO-8601 format date-time], indicating the time the event was detected or generated.
 Component::     An *optional* value indicating the component of the source machine that is responsible for the event, for example `mysql` or `eth0`.
 Group::         An *optional* value indicating the logical grouping of components of a service, for example `app-stack`.

docs/management/connectors/action-types/servicenow-sir.asciidoc

@@ -72,13 +72,11 @@ image::management/connectors/images/servicenow-sir-params-test.png[ServiceNow Se
 ServiceNow SecOps actions have the following configuration properties.
 
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
-Source Ips::           A list of source IPs related to the incident. The IPs will be added as observables to the security incident.
-Destination Ips::      A list of destination IPs related to the incident. The IPs will be added as observables to the security incident.
-Malware URLs::         A list of malware URLs related to the incident. The URLs will be added as observables to the security incident.
-Malware Hashes::       A list of malware hashes related to the incident. The hashes  will be added as observables to the security incident.
 Priority::             The priority of the incident.
 Category::             The category of the incident.
 Subcategory::          The subcategory of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow security incident. If an incident exists in ServiceNow with the same correlation ID the security incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
 

docs/management/connectors/action-types/servicenow.asciidoc

@@ -76,6 +76,8 @@ Severity::             The severity of the incident.
 Impact::               The effect an incident has on business. Can be measured by the number of affected users or by how critical it is to the business in question.
 Category::             The category of the incident.
 Subcategory::          The category of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow incident. If an incident exists in ServiceNow with the same correlation ID the incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.

docs/settings/spaces-settings.asciidoc

@@ -7,11 +7,6 @@
 
 By default, spaces is enabled in {kib}. To secure spaces, <<security-settings-kb,enable security>>.
 
-`xpack.spaces.enabled`::
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported and it will not be possible to disable this plugin."]
-To enable spaces, set to `true`. 
-The default is `true`.
-
 `xpack.spaces.maxSpaces`::
 The maximum number of spaces that you can use with the {kib} instance. Some {kib} operations
 return all spaces using a single `_search` from {es}, so you must

docs/spaces/index.asciidoc

@@ -109,11 +109,6 @@ image::images/spaces-configure-landing-page.png["Configure space-level landing p
 
 [float]
 [[spaces-delete-started]]
-=== Disable and version updates
-
-Spaces are automatically enabled in {kib}. If you don't want use this feature,
-you can disable it. For more information, refer to <<spaces-settings-kb,Spaces settings in {kib}>>.
-
-When you upgrade {kib}, the default space contains all of your existing saved objects.
-
+=== Disabling spaces
 
+Starting in {kib} 8.0, the Spaces feature cannot be disabled.

packages/elastic-apm-generator/src/lib/apm_error.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from './entity';
+import { Serializable } from './serializable';
+import { generateLongId, generateShortId } from './utils/generate_id';
+
+export class ApmError extends Serializable {
+  constructor(fields: Fields) {
+    super({
+      ...fields,
+      'processor.event': 'error',
+      'processor.name': 'error',
+      'error.id': generateShortId(),
+    });
+  }
+
+  serialize() {
+    const [data] = super.serialize();
+    data['error.grouping_key'] = generateLongId(
+      this.fields['error.grouping_name'] || this.fields['error.exception']?.[0]?.message
+    );
+    return [data];
+  }
+}

packages/elastic-apm-generator/src/lib/base_span.ts

@@ -10,7 +10,7 @@ import { Fields } from './entity';
 import { Serializable } from './serializable';
 import { Span } from './span';
 import { Transaction } from './transaction';
-import { generateTraceId } from './utils/generate_id';
+import { generateLongId } from './utils/generate_id';
 
 export class BaseSpan extends Serializable {
   private readonly _children: BaseSpan[] = [];
@@ -19,7 +19,7 @@ export class BaseSpan extends Serializable {
     super({
       ...fields,
       'event.outcome': 'unknown',
-      'trace.id': generateTraceId(),
+      'trace.id': generateLongId(),
       'processor.name': 'transaction',
     });
   }