Merge branch 'main' of github.com:elastic/kibana into fetch-ecs-in-ac…

…tions

.buildkite/scripts/steps/cloud/purge.js

@@ -50,6 +50,9 @@ for (const deployment of deploymentsToPurge) {
  console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
  try {
  execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
+ execSync(`vault delete secret/kibana-issues/dev/cloud-deploy/${deployment.name}`, {
+ stdio: 'inherit',
+ });
  } catch (ex) {
  console.error(ex.toString());
  }

docs/redirects.asciidoc

@@ -386,12 +386,17 @@ This content has moved. Refer to <<managing-data-views>>.
 
 This content has moved. Refer to <<kibana-role-management>>.
 
-[role="exclude" logging-configuration-changes]
-== Logging configuration changes 
+[role="exclude",id="logging-configuration-changes"]
+== Logging configuration changes
 
 This content has moved. Refer to <<logging-config-changes>>.
 
-[role="exclude" upgrade-migrations]
+[role="exclude",id="upgrade-migrations"]
 == Upgrade migrations
 
 This content has moved. Refer to <<saved-object-migrations>>.
+
+[role="exclude",id="upgrade-assistant"]
+== Upgrade Assistant
+
+This content has moved. Refer to {kibana-ref-all}/7.17/upgrade-assistant.html[Upgrade Assistant].

docs/setup/upgrade/upgrade-migrations.asciidoc

@@ -15,11 +15,11 @@ WARNING: The following instructions assumes {kib} is using the default index nam
 [[upgrade-migrations-process]]
 ==== Background
 
-Saved objects are stored in two indices: 
+Saved objects are stored in two indices:
 
 * `.kibana_{kibana_version}_001`, e.g. for Kibana v7.12.0 `.kibana_7.12.0_001`.
 * `.kibana_task_manager_{kibana_version}_001`, e.g. for Kibana v7.12.0 `.kibana_task_manager_7.12.0_001`.
- 
+
 The index aliases `.kibana` and `.kibana_task_manager` will always point to
 the most up-to-date saved object indices.
 
@@ -29,18 +29,18 @@ The first time a newer {kib} starts, it will first perform an upgrade migration
 [options="header"]
 |=======================
 |Upgrading from version | Outdated index (alias)
-| 6.0.0 through 6.4.x | `.kibana` 
+| 6.0.0 through 6.4.x | `.kibana`
 
 `.kibana_task_manager_7.12.0_001` (`.kibana_task_manager` alias)
 | 6.5.0 through 7.3.x | `.kibana_N` (`.kibana` alias)
-| 7.4.0 through 7.11.x 
-| `.kibana_N` (`.kibana` alias) 
+| 7.4.0 through 7.11.x
+| `.kibana_N` (`.kibana` alias)
 
 `.kibana_task_manager_N` (`.kibana_task_manager` alias)
 |=======================
 
 ==== Upgrading multiple {kib} instances
-When upgrading several {kib} instances connected to the same {es} cluster, ensure that all outdated instances are shutdown before starting the upgrade. 
+When upgrading several {kib} instances connected to the same {es} cluster, ensure that all outdated instances are shutdown before starting the upgrade.
 
 Kibana does not support rolling upgrades. However, once outdated instances are shutdown, all upgraded instances can be started in parallel in which case all instances will participate in the upgrade migration in parallel.
 
@@ -64,13 +64,15 @@ Error: Unable to complete saved object migrations for the [.kibana] index. Pleas
 Error: Unable to complete saved object migrations for the [.kibana] index. Please check the health of your Elasticsearch cluster and try again. Error: [timeout_exception]: Timed out waiting for completion of [org.elasticsearch.index.reindex.BulkByScrollTask@6a74c54]
 --------------------------------------------
 
-See https://github.com/elastic/kibana/issues/95321 for instructions to work around this issue.
- 
+Instructions to work around this issue are in https://github.com/elastic/kibana/issues/95321[this GitHub issue].
+
 [float]
 ===== Corrupt saved objects
 We highly recommend testing your {kib} upgrade in a development cluster to discover and remedy problems caused by corrupt documents, especially when there are custom integrations creating saved objects in your environment.
 
-Saved objects that were corrupted through manual editing or integrations will cause migration failures with a log message like `Failed to transform document. Transform: index-pattern:7.0.0\n Doc: {...}` or `Unable to migrate the corrupt Saved Object document ...`. Corrupt documents will have to be fixed or deleted before an upgrade migration can succeed.
+Saved objects that were corrupted through manual editing or integrations will cause migration
+failures with a log message like `Unable to migrate the corrupt Saved Object document ...`.
+Corrupt documents will have to be fixed or deleted before an upgrade migration can succeed.
 
 For example, given the following error message:
 
@@ -101,7 +103,7 @@ DELETE .kibana_7.12.0_001/_doc/marketing_space:dashboard:e3c5fc71-ac71-4805-bcab
 --------------------------------------------
 
 . Restart {kib}.
-
++
 In this example, the Dashboard with ID `e3c5fc71-ac71-4805-bcab-2bcc9cc93275` that belongs to the space `marketing_space` **will no longer be available**.
 
 Be sure you have a snapshot before you delete the corrupt document. If restoring from a snapshot is not an option, it is recommended to also delete the `temp` and `target` indices the migration created before restarting {kib} and retrying.
@@ -112,15 +114,16 @@ Matching index templates which specify `settings.refresh_interval` or `mappings`
 
 Prevention: narrow down the index patterns of any user-defined index templates to ensure that these won't apply to new `.kibana*` indices.
 
-Note: {kib} < 6.5 creates it's own index template called `kibana_index_template:.kibana` and index pattern `.kibana`. This index template will not interfere and does not need to be changed or removed.
+NOTE: {kib} < 6.5 creates it's own index template called `kibana_index_template:.kibana`
+and uses an index pattern of `.kibana`. This index template will not interfere and does not need to be changed or removed.
 
 [float]
 ===== An unhealthy {es} cluster
 Problems with your {es} cluster can prevent {kib} upgrades from succeeding. Ensure that your cluster has:
 
- * enough free disk space, at least twice the amount of storage taken up by the `.kibana` and `.kibana_task_manager` indices
- * sufficient heap size
- * a "green" cluster status
+ * Enough free disk space, at least twice the amount of storage taken up by the `.kibana` and `.kibana_task_manager` indices
+ * Sufficient heap size
+ * A "green" cluster status
 
 [float]
 ===== Different versions of {kib} connected to the same {es} index
@@ -134,20 +137,32 @@ For {kib} versions prior to 7.5.1, if the task manager index is set to `.tasks`
 [[resolve-migrations-failures]]
 ==== Resolving migration failures
 
-If {kib} terminates unexpectedly while migrating a saved object index it will automatically attempt to perform the migration again once the process has restarted. Do not delete any saved objects indices to attempt to fix a failed migration. Unlike previous versions, {kib} version 7.12.0 and later does not require deleting any indices to release a failed migration lock.
+If {kib} terminates unexpectedly while migrating a saved object index it will automatically attempt to
+perform the migration again once the process has restarted. Do not delete any saved objects indices to
+attempt to fix a failed migration. Unlike previous versions, {kib} version 7.12.0 and
+later does not require deleting any indices to release a failed migration lock.
 
-If upgrade migrations fail repeatedly, follow the advice in (preventing migration failures)[preventing-migration-failures]. Once the root cause for the migration failure has been addressed, {kib} will automatically retry the migration without any further intervention. If you're unable to resolve a failed migration following these steps, please contact support.
+If upgrade migrations fail repeatedly, follow the advice in
+<<preventing-migration-failures, preventing migration failures>>.
+Once the root cause for the migration failure has been addressed,
+{kib} will automatically retry the migration without any further intervention.
+If you're unable to resolve a failed migration following these steps, please contact support.
 
 [float]
 [[upgrade-migrations-rolling-back]]
 ==== Rolling back to a previous version of {kib}
 
-If you've followed the advice in (preventing migration failures)[preventing-migration-failures] and (resolving migration failures)[resolve-migrations-failures] and {kib} is still not able to upgrade successfully, you might choose to rollback {kib} until you're able to identify and fix the root cause.
+If you've followed the advice in <<preventing-migration-failures, preventing migration failures>>
+and <<resolve-migrations-failures, resolving migration failures>> and
+{kib} is still not able to upgrade successfully, you might choose to rollback {kib} until
+you're able to identify and fix the root cause.
 
-WARNING: Before rolling back {kib}, ensure that the version you wish to rollback to is compatible with your {es} cluster. If the version you're rolling back to is not compatible, you will have to also rollback {es}. +
+WARNING: Before rolling back {kib}, ensure that the version you wish to rollback to is compatible with
+your {es} cluster. If the version you're rolling back to is not compatible, you will have to also rollback {es}.
 Any changes made after an upgrade will be lost when rolling back to a previous version.
 
-In order to rollback after a failed upgrade migration, the saved object indices have to be rolled back to be compatible with the previous {kibana} version. 
+In order to rollback after a failed upgrade migration, the saved object indices have to be
+rolled back to be compatible with the previous {kib} version.
 
 [float]
 ===== Rollback by restoring a backup snapshot:
@@ -164,8 +179,11 @@ In order to rollback after a failed upgrade migration, the saved object indices
 
 1. Shutdown all {kib} instances to be 100% sure that there are no {kib} instances currently performing a migration.
 2. {ref}/snapshots-take-snapshot.html[Take a snapshot] that includes the `kibana` feature state. Snapshots include this feature state by default.
-3. Delete the version specific indices created by the failed upgrade migration. E.g. if you wish to rollback from a failed upgrade to v7.12.0 `DELETE /.kibana_7.12.0_*,.kibana_task_manager_7.12.0_*`
-4. Inspect the output of `GET /_cat/aliases`. If either the `.kibana` and/or `.kibana_task_manager` alias is missing, these will have to be created manually. Find the latest index from the output of `GET /_cat/indices` and create the missing alias to point to the latest index. E.g. if the `.kibana` alias was missing and the latest index is `.kibana_3` create a new alias with `POST /.kibana_3/_aliases/.kibana`.
+3. Delete the version specific indices created by the failed upgrade migration. For example, if you wish to rollback from a failed upgrade to v7.12.0 `DELETE /.kibana_7.12.0_*,.kibana_task_manager_7.12.0_*`
+4. Inspect the output of `GET /_cat/aliases`.
+If either the `.kibana` and/or `.kibana_task_manager` alias is missing, these will have to be created manually.
+Find the latest index from the output of `GET /_cat/indices` and create the missing alias to point to the latest index.
+For example. if the `.kibana` alias was missing and the latest index is `.kibana_3` create a new alias with `POST /.kibana_3/_aliases/.kibana`.
 5. Remove the write block from the rollback indices. `PUT /.kibana,.kibana_task_manager/_settings {"index.blocks.write": false}`
 6. Start up {kib} on the older version you wish to rollback to.
 

docs/user/management.asciidoc

@@ -167,10 +167,6 @@ set the timespan for notification messages, and much more.
 the full list of features that are included in your license,
 see the https://www.elastic.co/subscriptions[subscription page].
 
-| <<upgrade-assistant, Upgrade Assistant>>
-| Identify the issues that you need to address before upgrading to the
-next major version of {es}, and then reindex, if needed.
-
 |===
 
 
@@ -197,6 +193,4 @@ include::{kib-repo-dir}/spaces/index.asciidoc[]
 
 include::{kib-repo-dir}/management/managing-tags.asciidoc[]
 
-include::{kib-repo-dir}/management/upgrade-assistant/index.asciidoc[]
-
 include::{kib-repo-dir}/management/watcher-ui/index.asciidoc[]

src/dev/build/tasks/download_cloud_dependencies.ts

@@ -36,12 +36,19 @@ export const DownloadCloudDependencies: Task = {
 
  let buildId = '';
  if (!config.isRelease) {
- const manifest = await Axios.get(
- `https://artifacts-api.elastic.co/v1/versions/${config.getBuildVersion()}/builds/latest`
- );
- buildId = manifest.data.build.build_id;
+ const manifestUrl = `https://artifacts-api.elastic.co/v1/versions/${config.getBuildVersion()}/builds/latest`;
+ try {
+ const manifest = await Axios.get(manifestUrl);
+ buildId = manifest.data.build.build_id;
+ } catch (e) {
+ log.error(
+ `Unable to find Elastic artifacts for ${config.getBuildVersion()} at ${manifestUrl}.`
+ );
+ throw e;
+ }
  }
  await del([config.resolveFromRepo('.beats')]);
+
  await downloadBeat('metricbeat', buildId);
  await downloadBeat('filebeat', buildId);
  },

src/plugins/dashboard/kibana.json

@@ -8,6 +8,7 @@
  "version": "kibana",
  "requiredPlugins": [
  "data",
+ "dataViews",
  "embeddable",
  "controls",
  "inspector",

src/plugins/dashboard/public/application/dashboard_router.tsx

@@ -110,7 +110,7 @@ export async function mountApp({
  uiSettings: coreStart.uiSettings,
  scopedHistory: () => scopedHistory,
  screenshotModeService: screenshotMode,
- indexPatterns: dataStart.indexPatterns,
+ dataViews: dataStart.dataViews,
  savedQueryService: dataStart.query.savedQueries,
  savedObjectsClient: coreStart.savedObjects.client,
  savedDashboards: dashboardStart.getSavedDashboardLoader(),
@@ -212,7 +212,7 @@ export async function mountApp({
  .getIncomingEmbeddablePackage(DashboardConstants.DASHBOARDS_ID, false)
  );
  if (!hasEmbeddableIncoming) {
- dataStart.indexPatterns.clearCache();
+ dataStart.dataViews.clearCache();
  }
 
  // dispatch synthetic hash change event to update hash history objects

src/plugins/dashboard/public/application/hooks/use_dashboard_app_state.test.tsx

@@ -24,13 +24,15 @@ import { EmbeddableFactory, ViewMode } from '../../services/embeddable';
 import { dashboardStateStore, setDescription, setViewMode } from '../state';
 import { DashboardContainerServices } from '../embeddable/dashboard_container';
 import { createKbnUrlStateStorage, defer } from '../../../../kibana_utils/public';
-import { Filter, IIndexPattern, IndexPatternsContract } from '../../services/data';
 import { useDashboardAppState, UseDashboardStateProps } from './use_dashboard_app_state';
 import {
  getSampleDashboardInput,
  getSavedDashboardMock,
  makeDefaultServices,
 } from '../test_helpers';
+import { DataViewsContract } from '../../services/data';
+import { DataView } from '../../services/data_views';
+import type { Filter } from '@kbn/es-query';
 
 interface SetupEmbeddableFactoryReturn {
  finalizeEmbeddableCreation: () => void;
@@ -56,12 +58,10 @@ const createDashboardAppStateProps = (): UseDashboardStateProps => ({
 
 const createDashboardAppStateServices = () => {
  const defaults = makeDefaultServices();
- const indexPatterns = {} as IndexPatternsContract;
- const defaultIndexPattern = { id: 'foo', fields: [{ name: 'bar' }] } as IIndexPattern;
- indexPatterns.ensureDefaultDataView = jest.fn().mockImplementation(() => Promise.resolve(true));
- indexPatterns.getDefault = jest
- .fn()
- .mockImplementation(() => Promise.resolve(defaultIndexPattern));
+ const dataViews = {} as DataViewsContract;
+ const defaultDataView = { id: 'foo', fields: [{ name: 'bar' }] } as DataView;
+ dataViews.ensureDefaultDataView = jest.fn().mockImplementation(() => Promise.resolve(true));
+ dataViews.getDefault = jest.fn().mockImplementation(() => Promise.resolve(defaultDataView));
 
  const data = dataPluginMock.createStartContract();
  data.query.filterManager.getUpdates$ = jest.fn().mockImplementation(() => of(void 0));
@@ -71,7 +71,7 @@ const createDashboardAppStateServices = () => {
  .fn()
  .mockImplementation(() => of(void 0));
 
- return { ...defaults, indexPatterns, data };
+ return { ...defaults, dataViews, data };
 };
 
 const setupEmbeddableFactory = (

src/plugins/dashboard/public/application/hooks/use_dashboard_app_state.ts

@@ -15,6 +15,7 @@ import { BehaviorSubject, combineLatest, Observable, Subject } from 'rxjs';
 import { DashboardConstants } from '../..';
 import { ViewMode } from '../../services/embeddable';
 import { useKibana } from '../../services/kibana_react';
+import { DataView } from '../../services/data_views';
 import { getNewDashboardTitle } from '../../dashboard_strings';
 import { IKbnUrlStateStorage } from '../../services/kibana_utils';
 import { setDashboardState, useDashboardDispatch, useDashboardSelector } from '../state';
@@ -30,7 +31,7 @@ import {
  tryDestroyDashboardContainer,
  syncDashboardContainerInput,
  savedObjectToDashboardState,
- syncDashboardIndexPatterns,
+ syncDashboardDataViews,
  syncDashboardFilterState,
  loadSavedDashboardState,
  buildDashboardContainer,
@@ -81,7 +82,7 @@ export const useDashboardAppState = ({
  core,
  chrome,
  embeddable,
- indexPatterns,
+ dataViews,
  usageCollection,
  savedDashboards,
  initializerContext,
@@ -121,7 +122,7 @@ export const useDashboardAppState = ({
  search,
  history,
  embeddable,
- indexPatterns,
+ dataViews,
  notifications,
  kibanaVersion,
  savedDashboards,
@@ -234,11 +235,11 @@ export const useDashboardAppState = ({
  /**
  * Start syncing index patterns between the Query Service and the Dashboard Container.
  */
- const indexPatternsSubscription = syncDashboardIndexPatterns({
+ const dataViewsSubscription = syncDashboardDataViews({
  dashboardContainer,
- indexPatterns: dashboardBuildContext.indexPatterns,
- onUpdateIndexPatterns: (newIndexPatterns) =>
- setDashboardAppState((s) => ({ ...s, indexPatterns: newIndexPatterns })),
+ dataViews: dashboardBuildContext.dataViews,
+ onUpdateDataViews: (newDataViews: DataView[]) =>
+ setDashboardAppState((s) => ({ ...s, dataViews: newDataViews })),
  });
 
  /**
@@ -339,7 +340,7 @@ export const useDashboardAppState = ({
  stopWatchingAppStateInUrl();
  stopSyncingDashboardFilterState();
  lastSavedSubscription.unsubscribe();
- indexPatternsSubscription.unsubscribe();
+ dataViewsSubscription.unsubscribe();
  tryDestroyDashboardContainer(dashboardContainer);
  setDashboardAppState((state) => ({
  ...state,
@@ -368,7 +369,7 @@ export const useDashboardAppState = ({
  usageCollection,
  scopedHistory,
  notifications,
- indexPatterns,
+ dataViews,
  kibanaVersion,
  embeddable,
  docTitle,

src/plugins/dashboard/public/application/lib/build_dashboard_container.ts

@@ -31,7 +31,7 @@ import {
 } from '../../services/embeddable';
 
 type BuildDashboardContainerProps = DashboardBuildContext & {
- data: DashboardAppServices['data']; // the whole data service is required here because it is required by getUrlGeneratorState
+ data: DashboardAppServices['data']; // the whole data service is required here because it is required by getLocatorParams
  savedDashboard: DashboardSavedObject;
  initialDashboardState: DashboardState;
  incomingEmbeddable?: EmbeddablePackageState;