Merge branch 'master' into ml-base-privilege

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -27,6 +27,8 @@ for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
 }
 
+inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
+
 const pipeline = {
  steps: [
  {

.github/CODEOWNERS

@@ -95,15 +95,15 @@
 # Observability Shared
 /x-pack/plugins/observability/ @elastic/observability-ui
 
-# Logs and Metrics (Infra)
-/x-pack/plugins/infra/ @elastic/logs-metrics-ui
-/x-pack/plugins/infra/server/routes/log* @elastic/logs-metrics-ui @elastic/logs-ui
-/x-pack/plugins/infra/public/pages/logs/ @elastic/logs-metrics-ui @elastic/logs-ui
-/x-pack/plugins/infra/server/routes/metrics* @elastic/logs-metrics-ui @elastic/metrics-ui
-/x-pack/plugins/infra/public/pages/metrics/ @elastic/logs-metrics-ui @elastic/metrics-ui
+# Infra Monitoring
+/x-pack/plugins/infra/ @elastic/infra-monitoring-ui
+/x-pack/test/functional/apps/infra @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/infra @elastic/infra-monitoring-ui
 
-# Stack Monitoring
-/x-pack/plugins/monitoring/ @elastic/stack-monitoring-ui @elastic/logs-metrics-ui @elastic/metrics-ui
+# Elastic Stack Monitoring
+/x-pack/plugins/monitoring/ @elastic/infra-monitoring-ui
+/x-pack/test/functional/apps/monitoring @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
 
 # Fleet
 /x-pack/plugins/fleet/ @elastic/fleet
@@ -289,6 +289,7 @@
 /src/core/server/csp/ @elastic/kibana-security @elastic/kibana-core
 /src/plugins/interactive_setup/ @elastic/kibana-security
 /test/interactive_setup_api_integration/ @elastic/kibana-security
+/test/interactive_setup_functional/ @elastic/kibana-security
 /x-pack/plugins/spaces/ @elastic/kibana-security
 /x-pack/plugins/encrypted_saved_objects/ @elastic/kibana-security
 /x-pack/plugins/security/ @elastic/kibana-security

.github/workflows/add-to-apm-project.yml

@@ -0,0 +1,28 @@
+name: Add to APM Project
+on:
+ issues:
+ types:
+ - labeled
+jobs:
+ add_to_project:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: octokit/graphql-action@v2.x
+ id: add_to_project
+ if: |
+ github.event.label.name == 'Team:apm'
+ with:
+ headers: '{"GraphQL-Features": "projects_next_graphql"}'
+ query: |
+ mutation add_to_project($projectid:String!,$contentid:String!) {
+ addProjectNextItem(input:{projectId:$projectid contentId:$contentid}) {
+ projectNextItem {
+ id
+ }
+ }
+ }
+ projectid: ${{ env.PROJECT_ID }}
+ contentid: ${{ github.event.issue.node_id }}
+ env:
+ PROJECT_ID: "PN_kwDOAGc3Zs0VSg"
+ GITHUB_TOKEN: ${{ secrets.APM_TECH_KIBANA_USER_TOKEN }}

docs/api/saved-objects/bulk_create.asciidoc

@@ -66,7 +66,8 @@ Saved objects that are unable to persist are replaced with an error object.
 ==== Response code
 
 `200`::
- Indicates a successful call.
+ Indicates a successful call. Note, this HTTP response code indicates that the bulk operation succeeded. Errors pertaining to individual
+ objects will be returned in the response body. See the example below for details.
 
 [[saved-objects-api-bulk-create-example]]
 ==== Example
@@ -122,3 +123,19 @@ The API returns the following:
 --------------------------------------------------
 
 There is already a saved object with the `my-dashboard` ID, so only the index pattern is created.
+
+[[saved-objects-api-bulk-create-conflict-errors]]
+==== Conflict errors
+
+Starting in {kib} 8.0, saved objects can exist in multiple spaces. As a result, you may encounter different types of conflict errors when
+attempting to create an object:
+
+* *Regular conflict*: This is a 409 error without any metadata. It means an object of that type/ID already exists. This can be
+ overridden by using the `overwrite: true` option.
+* *Unresolvable conflict*: This is a 409 error with `isNotOverwritable: true` in its metadata. It means an object of that type/ID already
+ exists in a different space, and it cannot be overridden with the given parameters. To successfully overwrite this object, you must do so
+ in at least one space where it exists. You can specify that using the `space_id` path parameter _or_ the `initialNamespaces` parameter.
+* *Alias conflict*: This is a 409 error with a `spacesWithConflictingAliases` string array in its metadata. It means a conflicting
+ <<legacy-url-aliases,legacy URL alias>> for this type/ID exists in the space(s) where you attempted to create this object. A conflicting
+ legacy URL alias is one that points to a different type/ID. To successfully create this object, you need to first use the
+ <<spaces-api-disable-legacy-url-aliases,`_disable_legacy_url_aliases`>> API to disable the problematic legacy URL alias(es).

docs/api/saved-objects/bulk_get.asciidoc

@@ -53,7 +53,8 @@ Saved objects that are unable to persist are replaced with an error object.
 ==== Response code
 
 `200`::
- Indicates a successful call.
+ Indicates a successful call. Note, this HTTP response code indicates that the bulk operation succeeded. Errors pertaining to individual
+ objects will be returned in the response body. See the example below for details.
 
 [[saved-objects-api-bulk-get-body-example]]
 ==== Example

docs/api/saved-objects/bulk_resolve.asciidoc

@@ -46,7 +46,8 @@ that "exactMatch" is the default outcome, and the outcome only changes if an ali
 ==== Response code
 
 `200`::
- Indicates a successful call.
+ Indicates a successful call. Note, this HTTP response code indicates that the bulk operation succeeded. Errors pertaining to individual
+ objects will be returned in the response body. See the example below for details.
 
 [[saved-objects-api-bulk-resolve-body-example]]
 ==== Example

docs/api/saved-objects/create.asciidoc

@@ -64,6 +64,9 @@ used to specify a single space, and the "All spaces" identifier (`'*'`) is not a
 `200`::
  Indicates a successful call.
 
+`409`::
+ Indicates a <<saved-objects-api-create-conflict-errors,conflict error>>.
+
 [[saved-objects-api-create-example]]
 ==== Example
 
@@ -93,3 +96,11 @@ The API returns the following:
 --------------------------------------------------
 
 <1> When `my-pattern` is unspecified in the path, a unique ID is automatically generated.
+
+[[saved-objects-api-create-conflict-errors]]
+==== Conflict errors
+
+Starting in {kib} 8.0, saved objects can exist in multiple spaces. As a result, you may encounter different types of conflict errors when
+attempting to create an object. If you encounter a 409 error that cannot be overridden by using the `overwrite: true` option, you are likely
+hitting a different type of conflict error. The Create API response is limited and does not include additional metadata. You can get more
+details about this error by using the <<saved-objects-api-bulk-create,Bulk create API>> instead.

docs/api/saved-objects/update.asciidoc

@@ -45,6 +45,12 @@ WARNING: When you update, attributes are not validated, which allows you to pass
 `200`::
  Indicates a successful call.
 
+`404`::
+ Indicates the object was not found.
+
+`409`::
+ Indicates a <<saved-objects-api-update-conflict-errors,conflict error>>.
+
 [[saved-objects-api-update-example]]
 ==== Example
 
@@ -74,3 +80,10 @@ The API returns the following:
  }
 }
 --------------------------------------------------
+
+[[saved-objects-api-update-conflict-errors]]
+==== Conflict errors
+
+Starting in {kib} 8.0, saved objects can exist in multiple spaces. As a result, you may encounter a 409 *alias conflict* error when using
+the `upsert` option. The Update API response is limited and does not include additional metadata. You can get more details about this error
+by using the <<saved-objects-api-bulk-create,Bulk create API>> instead.

docs/development/core/server/kibana-plugin-core-server.elasticsearchclient.md

@@ -9,9 +9,9 @@ Client used to query the elasticsearch cluster.
 <b>Signature:</b>
 
 ```typescript
-export declare type ElasticsearchClient = Omit<KibanaClient, 'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close'> & {
+export declare type ElasticsearchClient = Omit<KibanaClient, 'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close' | 'diagnostic'> & {
  transport: {
- request(params: TransportRequestParams, options?: TransportRequestOptions): TransportRequestPromise<ApiResponse>;
+ request<TResponse = unknown>(params: TransportRequestParams, options?: TransportRequestOptions): Promise<TransportResult<TResponse>>;
  };
 };
 ```

docs/development/core/server/kibana-plugin-core-server.elasticsearcherrordetails.error.md

@@ -0,0 +1,14 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md) &gt; [error](./kibana-plugin-core-server.elasticsearcherrordetails.error.md)
+
+## ElasticsearchErrorDetails.error property
+
+<b>Signature:</b>
+
+```typescript
+error?: {
+ type: string;
+ reason?: string;
+ };
+```

docs/development/core/server/kibana-plugin-core-server.elasticsearcherrordetails.md

@@ -0,0 +1,19 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md)
+
+## ElasticsearchErrorDetails interface
+
+
+<b>Signature:</b>
+
+```typescript
+export interface ElasticsearchErrorDetails 
+```
+
+## Properties
+
+| Property | Type | Description |
+| --- | --- | --- |
+| [error](./kibana-plugin-core-server.elasticsearcherrordetails.error.md) | <code>{</code><br/><code> type: string;</code><br/><code> reason?: string;</code><br/><code> }</code> | |
+

docs/development/core/server/kibana-plugin-core-server.md

@@ -71,6 +71,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 | [DeprecationsServiceSetup](./kibana-plugin-core-server.deprecationsservicesetup.md) | The deprecations service provides a way for the Kibana platform to communicate deprecated features and configs with its users. These deprecations are only communicated if the deployment is using these features. Allowing for a user tailored experience for upgrading the stack version.<!-- -->The Deprecation service is consumed by the upgrade assistant to assist with the upgrade experience.<!-- -->If a deprecated feature can be resolved without manual user intervention. Using correctiveActions.api allows the Upgrade Assistant to use this api to correct the deprecation upon a user trigger. |
 | [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
 | [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) | A limited set of Elasticsearch configuration entries exposed to the <code>preboot</code> plugins at <code>setup</code>. |
+| [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md) | |
 | [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) | |
 | [ElasticsearchServiceSetup](./kibana-plugin-core-server.elasticsearchservicesetup.md) | |
 | [ElasticsearchServiceStart](./kibana-plugin-core-server.elasticsearchservicestart.md) | |

docs/development/core/server/kibana-plugin-core-server.savedobjectserrorhelpers.md

@@ -18,7 +18,6 @@ export declare class SavedObjectsErrorHelpers
 | [createBadRequestError(reason)](./kibana-plugin-core-server.savedobjectserrorhelpers.createbadrequesterror.md) | <code>static</code> | |
 | [createConflictError(type, id, reason)](./kibana-plugin-core-server.savedobjectserrorhelpers.createconflicterror.md) | <code>static</code> | |
 | [createGenericNotFoundError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.creategenericnotfounderror.md) | <code>static</code> | |
-| [createGenericNotFoundEsUnavailableError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.creategenericnotfoundesunavailableerror.md) | <code>static</code> | |
 | [createIndexAliasNotFoundError(alias)](./kibana-plugin-core-server.savedobjectserrorhelpers.createindexaliasnotfounderror.md) | <code>static</code> | |
 | [createInvalidVersionError(versionInput)](./kibana-plugin-core-server.savedobjectserrorhelpers.createinvalidversionerror.md) | <code>static</code> | |
 | [createTooManyRequestsError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.createtoomanyrequestserror.md) | <code>static</code> | |

docs/setup/settings.asciidoc

@@ -350,11 +350,6 @@ When `includeElasticMapsService` is turned off, only tile layer configured by <<
 | `map.emsUrl:`
  | Specifies the URL of a self hosted <<elastic-maps-server,{hosted-ems}>>
 
-| `map.proxyElasticMapsServiceInMaps:`
- | deprecated:[7.14.0,"In 8.0 and later, this setting will no longer be supported."]
- Set to `true` to proxy all <<maps, Maps application>> Elastic Maps Service
-requests through the {kib} server. *Default: `false`*
-
 | [[tilemap-settings]] `map.tilemap.options.attribution:` {ess-icon}
  | The map attribution string.
 *Default: `"© [Elastic Maps Service](https://www.elastic.co/elastic-maps-service)"`*

package.json

@@ -100,8 +100,8 @@
  "@elastic/apm-rum-react": "^1.3.1",
  "@elastic/charts": "38.0.1",
  "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
- "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21",
- "@elastic/ems-client": "7.16.0",
+ "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.35",
+ "@elastic/ems-client": "8.0.0",
  "@elastic/eui": "40.0.0",
  "@elastic/filesaver": "1.1.2",
  "@elastic/maki": "6.3.0",
@@ -217,7 +217,7 @@
  "deep-freeze-strict": "^1.1.1",
  "deepmerge": "^4.2.2",
  "del": "^5.1.0",
- "elastic-apm-node": "^3.21.1",
+ "elastic-apm-node": "^3.23.0",
  "execa": "^4.0.2",
  "exit-hook": "^2.2.0",
  "expiry-js": "0.1.7",

packages/elastic-apm-synthtrace/README.md

@@ -93,23 +93,24 @@ const esEvents = toElasticsearchOutput([
 
 Via the CLI, you can upload scenarios, either using a fixed time range or continuously generating data. Some examples are available in in `src/scripts/examples`. Here's an example for live data:
 
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
 
 For a fixed time window:
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
 
-The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
+The script will try to automatically find bootstrapped APM indices. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
 
 The following options are supported:
-| Option | Description | Default |
-| -------------- | ------------------------------------------------------- | ------------ |
-| `--from` | The start of the time window. | `now - 15m` |
-| `--to` | The end of the time window. | `now` |
-| `--live` | Continously ingest data | `false` |
-| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
-| `--clean` | Clean APM indices before indexing new data. | `false` |
-| `--interval` | The interval at which to index data. | `10s` |
-| `--logLevel` | Log level. | `info` |
-| `--lookback` | The lookback window for which data should be generated. | `15m` |
-| `--target` | Elasticsearch target, including username/password. | **Required** |
-| `--workers` | Amount of simultaneously connected ES clients. | `1` |
+| Option | Description | Default |
+| ------------------| ------------------------------------------------------- | ------------ |
+| `--target` | Elasticsearch target, including username/password. | **Required** |
+| `--from` | The start of the time window. | `now - 15m` |
+| `--to` | The end of the time window. | `now` |
+| `--live` | Continously ingest data | `false` |
+| `--clean` | Clean APM indices before indexing new data. | `false` |
+| `--workers` | Amount of Node.js worker threads | `5` |
+| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
+| `--interval` | The interval at which to index data. | `10s` |
+| `--clientWorkers` | Number of simultaneously connected ES clients | `5` |
+| `--batchSize` | Number of documents per bulk index request | `1000` |
+| `--logLevel` | Log level. | `info` |

packages/elastic-apm-synthtrace/src/lib/output/to_elasticsearch_output.ts

@@ -22,6 +22,18 @@ export interface ElasticsearchOutputWriteTargets {
  metric: string;
 }
 
+const observerDefaults = getObserverDefaults();
+
+const esDocumentDefaults = {
+ ecs: {
+ version: '1.4',
+ },
+};
+
+// eslint-disable-next-line guard-for-in
+for (const key in observerDefaults) {
+ set(esDocumentDefaults, key, observerDefaults[key as keyof typeof observerDefaults]);
+}
 export function toElasticsearchOutput({
  events,
  writeTargets,
@@ -30,22 +42,25 @@ export function toElasticsearchOutput({
  writeTargets: ElasticsearchOutputWriteTargets;
 }): ElasticsearchOutput[] {
  return events.map((event) => {
- const values = {
- ...event,
-  ...getObserverDefaults(),
+ const values = {};
+
+ Object.assign(values, event, {
  '@timestamp': new Date(event['@timestamp']!).toISOString(),
  'timestamp.us': event['@timestamp']! * 1000,
- 'ecs.version': '1.4',
  'service.node.name':
  event['service.node.name'] || event['container.id'] || event['host.name'],
- };
+ });
 
  const document = {};
+
+ Object.assign(document, esDocumentDefaults);
+
  // eslint-disable-next-line guard-for-in
  for (const key in values) {
  const val = values[key as keyof typeof values];
  set(document, key, val);
  }
+
  return {
  _index: writeTargets[event['processor.event'] as keyof ElasticsearchOutputWriteTargets],
  _source: document,