Merge branch 'master' into feature-ingest-fix-api-key-id-escaping

.ci/Jenkinsfile_flaky

@@ -35,7 +35,7 @@ kibanaPipeline(timeoutMinutes: 180) {
  if (!IS_XPACK) {
  kibanaPipeline.buildOss()
  if (CI_GROUP == '1') {
- runbld("./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
+ runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
  }
  } else {
  kibanaPipeline.buildXpack()

.github/CODEOWNERS

@@ -177,17 +177,23 @@
 # Elasticsearch UI
 /src/plugins/console/ @elastic/es-ui
 /src/plugins/es_ui_shared/ @elastic/es-ui
-/x-pack/plugins/console_extensions/ @elastic/es-ui
 /x-pack/legacy/plugins/cross_cluster_replication/ @elastic/es-ui
 /x-pack/legacy/plugins/index_lifecycle_management/ @elastic/es-ui
 /x-pack/legacy/plugins/index_management/ @elastic/es-ui
 /x-pack/legacy/plugins/license_management/ @elastic/es-ui
-/x-pack/plugins/remote_clusters/ @elastic/es-ui
 /x-pack/legacy/plugins/rollup/ @elastic/es-ui
-/x-pack/plugins/searchprofiler/ @elastic/es-ui
-/x-pack/plugins/painless_lab/ @elastic/es-ui
 /x-pack/legacy/plugins/snapshot_restore/ @elastic/es-ui
 /x-pack/legacy/plugins/upgrade_assistant/ @elastic/es-ui
+/x-pack/plugins/console_extensions/ @elastic/es-ui
+/x-pack/plugins/es_ui_shared/ @elastic/es-ui
+/x-pack/plugins/grokdebugger/ @elastic/es-ui
+/x-pack/plugins/index_management/ @elastic/es-ui
+/x-pack/plugins/license_management/ @elastic/es-ui
+/x-pack/plugins/painless_lab/ @elastic/es-ui
+/x-pack/plugins/remote_clusters/ @elastic/es-ui
+/x-pack/plugins/rollup/ @elastic/es-ui
+/x-pack/plugins/searchprofiler/ @elastic/es-ui
+/x-pack/plugins/snapshot_restore/ @elastic/es-ui
 /x-pack/plugins/upgrade_assistant/ @elastic/es-ui
 /x-pack/plugins/watcher/ @elastic/es-ui
 

.github/paths-labeller.yml

@@ -1,13 +1,4 @@
 ---
- - "Team:AppArch":
- - "src/plugins/bfetch/**/*.*"
- - "src/plugins/dashboard_embeddable_container/**/*.*"
- - "src/plugins/data/**/*.*"
- - "src/plugins/embeddable/**/*.*"
- - "src/plugins/expressions/**/*.*"
- - "src/plugins/inspector/**/*.*"
- - "src/plugins/ui_actions/**/*.*"
- - "src/plugins/visualizations/**/*.*"
  - "Feature:Embedding":
  - "src/plugins/embeddable/**/*.*"
  - "src/plugins/dashboard_embeddable_container/**/*.*"

docs/apm/agent-configuration.asciidoc

@@ -31,37 +31,18 @@ Kibana communicates any changed settings to APM Server so that your agents only
 [float]
 ==== Supported configurations
 
-[float]
-===== `CAPTURE_BODY`
-
-added[7.5.0] Can be `"off"`, `"errors"`, `"transactions"`, or `"all"`. Defaults to `"off"`.
-
-For transactions that are HTTP requests, the Agent can optionally capture the request body, e.g., POST variables.
-Remember, request bodies often contain sensitive values like passwords, credit card numbers, etc.
-If your service handles sensitive data, enable this feature with care.
-Turning on body capturing can also significantly increase the overhead the overhead of the Agent,
-and the Elasticsearch index size.
-
-[float]
-===== `TRANSACTION_MAX_SPANS`
-
-added[7.5.0] A number between `0` and `32000`. Defaults to `500`.
-
-Limit the number of spans that are recorded per transaction.
-This is helpful in cases where a transaction creates a very high amount of spans, e.g., thousands of SQL queries.
-Setting an upper limit will help prevent the Agent and the APM Server from being overloaded.
-
-[float]
-===== `TRANSACTION_SAMPLE_RATE`
-
-added[7.3.0] A sample rate between `0.000` and `1.0`. Default configuration is `1.0` (100% of traces).
-
-Adjusting the sampling rate controls what percent of requests are traced.
-`1.0` means _all_ requests are traced. If you set the `TRANSACTION_SAMPLE_RATE` to a value below `1.0`,
-the agent will randomly sample only a subset of transactions.
-Unsampled transactions only record the name of the transaction, the overall transaction time, and the result.
-
-IMPORTANT: In a distributed trace, the sampling decision is propagated by the initializing Agent.
-This means if you're using multiple agents, only the originating service's sampling rate will be used.
-Be sure to set sensible defaults in _all_ of your agents, especially the
-{apm-rum-ref}/configuration.html#transaction-sample-rate[JavaScript RUM Agent].
+Each Agent has its own list of supported configurations.
+After selecting a Service name and environment in the APM app,
+a list of all available configuration options,
+including descriptions and default values, will be displayed.
+
+Supported configurations are also marked in each Agent's configuration documentation:
+
+[horizontal]
+Go Agent:: {apm-go-ref}/configuration.html[Configuration reference]
+Java Agent:: {apm-java-ref}/configuration.html[Configuration reference]
+.NET Agent:: {apm-dotnet-ref}/configuration.html[Configuration reference]
+Node.js Agent:: {apm-node-ref}/configuration.html[Configuration reference]
+Python Agent:: {apm-py-ref}/configuration.html[Configuration reference]
+Ruby Agent:: {apm-ruby-ref}/configuration.html[Configuration reference]
+Real User Monitoring (RUM) Agent:: {apm-rum-ref}/configuration.html[Configuration reference]

docs/canvas/canvas-elements.asciidoc

@@ -23,7 +23,7 @@ By default, most of the elements you create use demo data until you change the d
 
 * *{es} SQL* — Access your data in {es} using SQL syntax. For information about SQL syntax, refer to {ref}/sql-spec.html[SQL language].
 
-* *{es} raw data* — Access your raw data in {es} without the use of aggregations. Use {es} raw data when you have low volume datasets, or to plot exact, non-aggregated values.
+* *{es} documents* &mdash; Access your data in {es} without using aggregations. To use, select an index and fields, and optionally enter a query using the <<lucene-query,Lucene Query Syntax>>. Use the *{es} documents* data source when you have low volume datasets, to view raw documents, or to plot exact, non-aggregated values on a chart.
 
 * *Timelion* &mdash; Access your time series data using <<timelion,Timelion>> queries. To use Timelion queries, you can enter a query using the <<lucene-query,Lucene Query Syntax>>.
 

docs/epm/index.asciidoc → docs/ingest_manager/index.asciidoc

@@ -1,8 +1,8 @@
 [role="xpack"]
 [[epm]]
-== Elastic Package Manager
+== Ingest Manager
 
-These are the docs for the Elastic Package Manager (EPM).
+These are the docs for the Ingest Manager.
 
 
 === Configuration
@@ -39,16 +39,71 @@ curl -X DELETE localhost:5601/api/ingest_manager/epm/packages/iptables-1.0.4
 
 This section is to define terms used across ingest management.
 
+==== Data Source
+
+A data source is a definition on how to collect data from a service, for example `nginx`. A data source contains
+definitions for one or multiple inputs and each input can contain one or multiple streams.
+
+With the example of the nginx Data Source, it contains to inputs: `logs` and `nginx/metrics`. Logs and metrics are collected
+differently. The `logs` input contains two streams, `access` and `error`, the `nginx/metrics` input contains the stubstatus stream.
+
+
+==== Data Stream
+
+Data Streams are a [new concept](https://github.com/elastic/elasticsearch/issues/53100) in Elasticsearch which simplify
+ingesting data and the setup of Elasticsearch.
+
 ==== Elastic Agent
+
 A single, unified agent that users can deploy to hosts or containers. It controls which data is collected from the host or containers and where the data is sent. It will run Beats, Endpoint or other monitoring programs as needed. It can operate standalone or pull a configuration policy from Fleet.
 
+
+==== Elastic Package Registry
+
+The Elastic Package Registry (EPR) is a service which runs under [https://epr.elastic.co]. It serves the packages through its API.
+More details about the registry can be found [here](https://github.com/elastic/package-registry).
+
+==== Fleet
+
+Fleet is the part of the Ingest Manager UI in Kibana that handles the part of enrolling Elastic Agents,
+managing agents and sending configurations to the Elastic Agent.
+
+==== Indexing Strategy
+
+Ingest Management + Elastic Agent follow a strict new indexing strategy: `{type}-{dataset}-{namespace}`. An example
+for this is `logs-nginx.access-default`. More details about it can be found in the Index Strategy below. All data of
+the index strategy is sent to Data Streams.
+
+==== Input
+
+An input is the configuration unit in an Agent Config that defines the options on how to collect data from 
+an endpoint. This could be username / password which are need to authenticate with a service or a host url 
+as an example.
+
+An input is part of a Data Source and contains streams.
+
+==== Integration
+
+An integration is a package with the type integration. An integration package has at least 1 data source
+and usually collects data from / about a service.
+
+
 ==== Namespace
+
 A user-specified string that will be used to part of the index name in Elasticsearch. It helps users identify logs coming from a specific environment (like prod or test), an application, or other identifiers.
 
+
 ==== Package
 
-A package contains all the assets for the Elastic Stack. A more detailed definition of a package can be found under https://github.com/elastic/package-registry.
+A package contains all the assets for the Elastic Stack. A more detailed definition of a 
+package can be found under https://github.com/elastic/package-registry.
+
+Besides the assets, a package contains the data source definitions with its inputs and streams.
+
+==== Stream
 
+A stream is a configuration unit in the Elastic Agent config. A stream is part of an input and defines how the data
+fetched by this input should be processed and which Data Stream to send it to.
 
 == Indexing Strategy
 

docs/logs/using.asciidoc

@@ -38,7 +38,7 @@ Log entries for the specified time appear in the middle of the page. To quickly
 [[logs-customize]]
 === Customize your view
 Click *Customize* to customize the view.
-Here, you can set the scale to use for the minimap timeline, choose whether to wrap long lines, and choose your preferred text size.
+Here, you can choose whether to wrap long lines, and choose your preferred text size.
 
 [float]
 === Configuring the data to use for your logs

docs/migration/migrate_8_0.asciidoc

@@ -61,24 +61,53 @@ for example, `logstash-*`.
 *Impact:* Use `xpack.security.authc.providers` instead.
 
 [float]
-==== `xpack.security.authc.saml.realm` is now mandatory when using the SAML authentication provider
-*Details:* Previously Kibana was choosing the appropriate Elasticsearch SAML realm automatically using the `Assertion Consumer Service`
-URL that it derived from the actual server address. Starting in 8.0.0, the Elasticsearch SAML realm name that Kibana will use should be
-specified explicitly.
+==== `xpack.security.authc.providers` has changed value format
+*Details:* `xpack.security.authc.providers` setting in the `kibana.yml` has changed value format.
+
+*Impact:* Array of provider types as a value is no longer supported, use extended object format instead.
+
+[float]
+==== `xpack.security.authc.saml` is no longer valid
+*Details:* The deprecated `xpack.security.authc.saml` setting in the `kibana.yml` file has been removed.
 
-*Impact:* Always define `xpack.security.authc.saml.realm` when using the SAML authentication provider.
+*Impact:* Configure SAML authentication providers using `xpack.security.authc.providers.saml.{provider unique name}.*` settings instead.
+
+[float]
+==== `xpack.security.authc.oidc` is no longer valid
+*Details:* The deprecated `xpack.security.authc.oidc` setting in the `kibana.yml` file has been removed.
+
+*Impact:* Configure OpenID Connect authentication providers using `xpack.security.authc.providers.oidc.{provider unique name}.*` settings instead.
 
 [float]
 ==== `xpack.security.public` is no longer valid
-*Details:* The deprecated `xpack.security.public` setting in the `kibana.yml` file has been removed.
+*Details:* Previously Kibana was choosing the appropriate Elasticsearch SAML realm automatically using the `Assertion Consumer Service`
+URL that it derived from the actual server address and `xpack.security.public` setting. Starting in 8.0.0, the deprecated `xpack.security.public` setting in the `kibana.yml` file has been removed and the Elasticsearch SAML realm name that Kibana will use should be specified explicitly.
 
-*Impact:* Define `xpack.security.authc.saml.realm` when using the SAML authentication provider instead.
+*Impact:* Define `xpack.security.authc.providers.saml.{provider unique name}.realm` when using the SAML authentication providers instead.
 
 [float]
 ==== `/api/security/v1/saml` endpoint is no longer supported
 *Details:* The deprecated `/api/security/v1/saml` endpoint is no longer supported.
 
-*Impact:* Rely on `/api/security/saml/callback` endpoint when using SAML instead. This change should be reflected in Kibana `server.xsrf.whitelist` config as well as in Elasticsearch and Identity Provider SAML settings.
+*Impact:* Rely on `/api/security/saml/callback` endpoint when using SAML instead. This change should be reflected in Elasticsearch and Identity Provider SAML settings.
+
+[float]
+==== `/api/security/v1/oidc` endpoint is no longer supported
+*Details:* The deprecated `/api/security/v1/oidc` endpoint is no longer supported.
+
+*Impact:* Rely on `/api/security/oidc/callback` endpoint when using OpenID Connect instead. This change should be reflected in Elasticsearch and OpenID Connect Provider settings.
+
+[float]
+==== `/api/security/v1/oidc` endpoint is no longer supported for Third Party initiated login
+*Details:* The deprecated `/api/security/v1/oidc` endpoint is no longer supported for Third Party initiated login.
+
+*Impact:* Rely on `/api/security/oidc/initiate_login` endpoint when using Third Party initiated OpenID Connect login instead. This change should be reflected in Elasticsearch and OpenID Connect Provider settings.
+
+[float]
+==== `/api/security/v1/oidc/implicit` endpoint is no longer supported
+*Details:* The deprecated `/api/security/v1/oidc/implicit` endpoint is no longer supported.
+
+*Impact:* Rely on `/api/security/oidc/implicit` endpoint when using OpenID Connect Implicit Flow instead. This change should be reflected in OpenID Connect Provider settings.
 
 [float]
 === `optimize` directory is now in the `data` folder
@@ -112,4 +141,13 @@ access level.
 been deprecated with warnings that have been logged throughout 7.x. Please use Kibana UI to re-generate the
 POST URL snippets if you depend on these for automated PDF reports.
 
+[float]
+=== Configurations starting with `xpack.telemetry` are no longer valid
+
+*Details:*
+The `xpack.` prefix has been removed for all telemetry configurations.
+
+*Impact:*
+For any configurations beginning with `xpack.telemetry`, remove the `xpack` prefix. Use {kibana-ref}/telemetry-settings-kbn.html#telemetry-general-settings[`telemetry.enabled`] instead.
+
 // end::notable-breaking-changes[]

docs/settings/telemetry-settings.asciidoc

@@ -0,0 +1,38 @@
+[[telemetry-settings-kbn]]
+=== Telemetry settings in Kibana
+++++
+<titleabbrev>Telemetry settings</titleabbrev>
+++++
+
+By default, Usage Collection (also known as Telemetry) is enabled. This
+helps us learn about the {kib} features that our users are most interested in, so we
+can focus our efforts on making them even better.
+
+You can control whether this data is sent from the {kib} servers, or if it should be sent 
+from the user's browser, in case a firewall is blocking the connections from the server. Additionally, you can decide to completely disable this feature either in the config file or in {kib} via *Management > Kibana > Advanced Settings > Usage Data*.
+
+See our https://www.elastic.co/legal/privacy-statement[Privacy Statement] to learn more.
+
+[float]
+[[telemetry-general-settings]]
+==== General telemetry settings
+
+`telemetry.enabled`:: *Default: true*.
+Set to `true` to send cluster statistics to Elastic. Reporting your
+cluster statistics helps us improve your user experience. Your data is never
+shared with anyone. Set to `false` to disable statistics reporting from any
+browser connected to the {kib} instance.
+
+`telemetry.sendUsageFrom`:: *Default: 'browser'*.
+Set to `'server'` to report the cluster statistics from the {kib} server.
+If the server fails to connect to our endpoint at https://telemetry.elastic.co/, it assumes
+it is behind a firewall and falls back to `'browser'` to send it from users' browsers
+when they are navigating through {kib}.
+
+`telemetry.optIn`:: *Default: true*.
+Set to `true` to automatically opt into reporting cluster statistics. You can also opt out through
+*Advanced Settings* in {kib}.
+
+`telemetry.allowChangingOptInStatus`:: *Default: true*.
+Set to `true` to allow overwriting the `telemetry.optIn` setting via the {kib} UI. 
+Note: When `false`, `telemetry.optIn` must be `true`. To disable telemetry and not allow users to change that parameter, use `telemetry.enabled`.

docs/setup/settings.asciidoc

@@ -410,9 +410,7 @@ all http requests to https over the port configured as `server.port`.
 supported protocols with versions. Valid protocols: `TLSv1`, `TLSv1.1`, `TLSv1.2`
 
 `server.xsrf.whitelist:`:: It is not recommended to disable protections for
-arbitrary API endpoints. Instead, supply the `kbn-xsrf` header. There are some
-scenarios where whitelisting is required, however, such as
-<<kibana-authentication, SAML and OpenID Connect Single Sign-On setups>>.
+arbitrary API endpoints. Instead, supply the `kbn-xsrf` header.
 The `server.xsrf.whitelist` setting requires the following format:
 
 [source,text]
@@ -465,3 +463,4 @@ include::{docdir}/settings/reporting-settings.asciidoc[]
 include::secure-settings.asciidoc[]
 include::{docdir}/settings/security-settings.asciidoc[]
 include::{docdir}/settings/spaces-settings.asciidoc[]
+include::{docdir}/settings/telemetry-settings.asciidoc[]