From 4dbc30bc084e2d7e2e0d2d9b0a81fcf82922d0c8 Mon Sep 17 00:00:00 2001
From: Constance <constancecchen@users.noreply.github.com>
Date: Thu, 24 Sep 2020 14:22:11 -0700
Subject: [PATCH 01/22] [Enterprise Search] Move externalUrl helper out of
 React Context (#78368)

* Add new/simpler externalUrl helper and initialize it on renderApp

- This uses a simple JS obj to store the enterpriseSearchUrl reference (once on plugin init)
  - This is vs. a class, which needs to be instantiated and passed around - the new obj can be imported flatly at any time
  - I also opted to not convert this into a Kea logic file - after some deliberation I decided against it because it felt really weird as one. It's not storing "state" per se that ever needs to be updated, it's simply a one-time set obj that contains helper functions.
  - There's also some hope that we might eventually not need this helper after the full migration, so the simpler it is to delete the better
- Uses a getter & setter to ensure that we don't accidentally mutate said obj after initialization

* Update all components using get*SearchUrl helpers

* Update tests for updated components

- Mostly just consists of mocking externalUrl and importing that mock

* Remove old ExternalUrl class/context

TODO in next commit: Address kibana_header_actions

* Update Workplace Search Header Actions to use new externalUrl helper

NOTE: this requires a temporary workaround of initializing externalUrl.enterpriseSearch in plugin.ts rather than in renderApp, because renderApp loads *after* the header app does.

I plan on fixing this in a future PR so that setHeaderActionMenu is called AFTER renderApp has done loading (to ensure all the state in headerActions we need is available).
---
 .../__mocks__/enterprise_search_url.mock.ts   |  9 ++++
 .../__mocks__/kibana_context.mock.ts          |  3 --
 .../components/empty_state.test.tsx           |  1 -
 .../components/empty_state.tsx                |  7 +---
 .../components/header.test.tsx                |  2 +-
 .../engine_overview/components/header.tsx     |  7 +---
 .../engine_overview/engine_table.test.tsx     |  2 +-
 .../engine_overview/engine_table.tsx          |  7 +---
 .../applications/app_search/index.test.tsx    |  1 +
 .../public/applications/app_search/index.tsx  |  5 +--
 .../public/applications/index.test.tsx        |  2 +-
 .../public/applications/index.tsx             | 18 +++-----
 .../external_url.test.ts                      | 42 +++++++++++++++++++
 .../enterprise_search_url/external_url.ts     | 37 ++++++++++++++++
 .../generate_external_url.test.ts             | 25 -----------
 .../generate_external_url.ts                  | 38 -----------------
 .../shared/enterprise_search_url/index.ts     |  7 +++-
 .../layout/kibana_header_actions.test.tsx     | 20 ++++-----
 .../layout/kibana_header_actions.tsx          | 11 ++---
 .../components/layout/nav.test.tsx            |  3 +-
 .../components/layout/nav.tsx                 |  8 +---
 .../product_button/product_button.test.tsx    |  1 -
 .../shared/product_button/product_button.tsx  |  7 +---
 .../views/overview/onboarding_card.test.tsx   |  2 +-
 .../views/overview/onboarding_card.tsx        |  7 +---
 .../views/overview/onboarding_steps.test.tsx  |  1 -
 .../views/overview/onboarding_steps.tsx       |  7 +---
 .../views/overview/recent_activity.test.tsx   |  1 -
 .../views/overview/recent_activity.tsx        |  7 +---
 .../views/overview/statistic_card.test.tsx    |  2 +-
 .../views/overview/statistic_card.tsx         |  9 +---
 .../enterprise_search/public/plugin.ts        | 17 ++++----
 32 files changed, 147 insertions(+), 169 deletions(-)
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/__mocks__/enterprise_search_url.mock.ts
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.test.ts
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.ts
 delete mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.test.ts
 delete mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.ts

diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/enterprise_search_url.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/enterprise_search_url.mock.ts
new file mode 100644
index 00000000000000..47660d0a317207
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/enterprise_search_url.mock.ts
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { externalUrl } from '../shared/enterprise_search_url';
+
+externalUrl.enterpriseSearchUrl = 'http://localhost:3002';
diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kibana_context.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kibana_context.mock.ts
index ea3c3923cc4720..ee77b0937cd82a 100644
--- a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kibana_context.mock.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kibana_context.mock.ts
@@ -4,8 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ExternalUrl } from '../shared/enterprise_search_url';
-
 /**
  * A set of default Kibana context values to use across component tests.
  * @see enterprise_search/public/index.tsx for the KibanaContext definition/import
@@ -15,5 +13,4 @@ export const mockKibanaContext = {
   setBreadcrumbs: jest.fn(),
   setDocTitle: jest.fn(),
   config: { host: 'http://localhost:3002' },
-  externalUrl: new ExternalUrl('http://localhost:3002'),
 };
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.test.tsx
index 233db7d4c59177..53f50822cf6535 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.test.tsx
@@ -5,7 +5,6 @@
  */
 
 import '../../../../__mocks__/kea.mock';
-import '../../../../__mocks__/shallow_usecontext.mock';
 
 import React from 'react';
 import { shallow } from 'enzyme';
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.tsx
index 5ed1f0b277306b..cfe88d00ce14eb 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/empty_state.tsx
@@ -4,15 +4,15 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { useValues } from 'kea';
 import { EuiPageContent, EuiEmptyPrompt, EuiButton } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 
 import { sendTelemetry } from '../../../../shared/telemetry';
 import { HttpLogic } from '../../../../shared/http';
+import { getAppSearchUrl } from '../../../../shared/enterprise_search_url';
 import { SetAppSearchChrome as SetPageChrome } from '../../../../shared/kibana_chrome';
-import { KibanaContext, IKibanaContext } from '../../../../index';
 import { CREATE_ENGINES_PATH } from '../../../routes';
 
 import { EngineOverviewHeader } from './header';
@@ -21,9 +21,6 @@ import './empty_state.scss';
 
 export const EmptyState: React.FC = () => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getAppSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const buttonProps = {
     href: getAppSearchUrl(CREATE_ENGINES_PATH),
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.test.tsx
index 8c7dfa2b7c3d63..78ee5764be5a97 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.test.tsx
@@ -5,7 +5,7 @@
  */
 
 import '../../../../__mocks__/kea.mock';
-import '../../../../__mocks__/shallow_usecontext.mock';
+import '../../../../__mocks__/enterprise_search_url.mock';
 
 import React from 'react';
 import { shallow } from 'enzyme';
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.tsx
index dca0d45a207b4a..6ebb2c5bf453da 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/header.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { useValues } from 'kea';
 import {
   EuiPageHeader,
@@ -18,13 +18,10 @@ import { FormattedMessage } from '@kbn/i18n/react';
 
 import { sendTelemetry } from '../../../../shared/telemetry';
 import { HttpLogic } from '../../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../../index';
+import { getAppSearchUrl } from '../../../../shared/enterprise_search_url';
 
 export const EngineOverviewHeader: React.FC = () => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getAppSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const buttonProps = {
     fill: true,
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.test.tsx
index 8e92f21f8ffeda..c66fd24fee12a8 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.test.tsx
@@ -5,7 +5,7 @@
  */
 
 import '../../../__mocks__/kea.mock';
-import '../../../__mocks__/shallow_usecontext.mock';
+import '../../../__mocks__/enterprise_search_url.mock';
 import { mockHttpValues } from '../../../__mocks__/';
 
 import React from 'react';
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.tsx
index 6888be1dc2b5b7..40fb313f30b31b 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/engine_table.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { useValues } from 'kea';
 import { EuiBasicTable, EuiBasicTableColumn, EuiLink } from '@elastic/eui';
 import { FormattedMessage, FormattedDate, FormattedNumber } from '@kbn/i18n/react';
@@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n';
 
 import { sendTelemetry } from '../../../shared/telemetry';
 import { HttpLogic } from '../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getAppSearchUrl } from '../../../shared/enterprise_search_url';
 import { getEngineRoute } from '../../routes';
 
 import { ENGINES_PAGE_SIZE } from '../../../../../common/constants';
@@ -43,9 +43,6 @@ export const EngineTable: React.FC<IEngineTableProps> = ({
   pagination: { totalEngines, pageIndex, onPaginate },
 }) => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getAppSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const engineLinkProps = (name: string) => ({
     href: getAppSearchUrl(getEngineRoute(name)),
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/index.test.tsx
index 350bc97085d7be..052f4446e4409b 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/index.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/index.test.tsx
@@ -6,6 +6,7 @@
 
 import '../__mocks__/shallow_usecontext.mock';
 import '../__mocks__/kea.mock';
+import '../__mocks__/enterprise_search_url.mock';
 
 import React, { useContext } from 'react';
 import { Redirect } from 'react-router-dom';
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/index.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/index.tsx
index c848415daf6123..410f6eb524822c 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/index.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/index.tsx
@@ -11,6 +11,7 @@ import { useActions, useValues } from 'kea';
 import { i18n } from '@kbn/i18n';
 
 import { KibanaContext, IKibanaContext } from '../index';
+import { getAppSearchUrl } from '../shared/enterprise_search_url';
 import { HttpLogic } from '../shared/http';
 import { AppLogic } from './app_logic';
 import { IInitialAppData } from '../../../common/types';
@@ -86,10 +87,6 @@ export const AppSearchConfigured: React.FC<IInitialAppData> = (props) => {
 };
 
 export const AppSearchNav: React.FC = () => {
-  const {
-    externalUrl: { getAppSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
-
   const {
     myRole: { canViewSettings, canViewAccountCredentials, canViewRoleMappings },
   } = useValues(AppLogic);
diff --git a/x-pack/plugins/enterprise_search/public/applications/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/index.test.tsx
index 6ee63ee22cae2b..66772f96671e81 100644
--- a/x-pack/plugins/enterprise_search/public/applications/index.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/index.test.tsx
@@ -54,7 +54,7 @@ describe('renderHeaderActions', () => {
     const mockHeaderEl = document.createElement('header');
     const MockHeaderActions = () => <button className="hello-world">Hello World</button>;
 
-    const unmount = renderHeaderActions(MockHeaderActions, mockHeaderEl, {} as any);
+    const unmount = renderHeaderActions(MockHeaderActions, mockHeaderEl);
     expect(mockHeaderEl.querySelector('.hello-world')).not.toBeNull();
 
     unmount();
diff --git a/x-pack/plugins/enterprise_search/public/applications/index.tsx b/x-pack/plugins/enterprise_search/public/applications/index.tsx
index 4a25ecf6067cc3..2c6bc787923e36 100644
--- a/x-pack/plugins/enterprise_search/public/applications/index.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/index.tsx
@@ -18,12 +18,11 @@ import { PluginsStart, ClientConfigType, ClientData } from '../plugin';
 import { mountLicensingLogic } from './shared/licensing';
 import { mountHttpLogic } from './shared/http';
 import { mountFlashMessagesLogic } from './shared/flash_messages';
-import { IExternalUrl } from './shared/enterprise_search_url';
+import { externalUrl } from './shared/enterprise_search_url';
 import { IInitialAppData } from '../../common/types';
 
 export interface IKibanaContext {
   config: { host?: string };
-  externalUrl: IExternalUrl;
   navigateToUrl: ApplicationStart['navigateToUrl'];
   setBreadcrumbs(crumbs: ChromeBreadcrumb[]): void;
   setDocTitle(title: string): void;
@@ -42,7 +41,8 @@ export const renderApp = (
   { params, core, plugins }: { params: AppMountParameters; core: CoreStart; plugins: PluginsStart },
   { config, data }: { config: ClientConfigType; data: ClientData }
 ) => {
-  const { externalUrl, errorConnecting, ...initialData } = data;
+  const { publicUrl, errorConnecting, ...initialData } = data;
+  externalUrl.enterpriseSearchUrl = publicUrl || config.host || '';
 
   resetContext({ createStore: true });
   const store = getContext().store as Store;
@@ -64,7 +64,6 @@ export const renderApp = (
       <KibanaContext.Provider
         value={{
           config,
-          externalUrl,
           navigateToUrl: core.application.navigateToUrl,
           setBreadcrumbs: core.chrome.setBreadcrumbs,
           setDocTitle: core.chrome.docTitle.change,
@@ -93,15 +92,8 @@ export const renderApp = (
  * a custom HeaderActions component (e.g., WorkplaceSearchHeaderActions)
  * @see https://github.com/elastic/kibana/blob/master/docs/development/core/public/kibana-plugin-core-public.appmountparameters.setheaderactionmenu.md
  */
-interface IHeaderActionsProps {
-  externalUrl: IExternalUrl;
-}
 
-export const renderHeaderActions = (
-  HeaderActions: React.FC<IHeaderActionsProps>,
-  kibanaHeaderEl: HTMLElement,
-  externalUrl: IExternalUrl
-) => {
-  ReactDOM.render(<HeaderActions externalUrl={externalUrl} />, kibanaHeaderEl);
+export const renderHeaderActions = (HeaderActions: React.FC, kibanaHeaderEl: HTMLElement) => {
+  ReactDOM.render(<HeaderActions />, kibanaHeaderEl);
   return () => ReactDOM.unmountComponentAtNode(kibanaHeaderEl);
 };
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.test.ts b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.test.ts
new file mode 100644
index 00000000000000..55c4f465d9ed42
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.test.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { externalUrl, getEnterpriseSearchUrl, getAppSearchUrl, getWorkplaceSearchUrl } from './';
+
+describe('Enterprise Search external URL helpers', () => {
+  describe('getter/setter tests', () => {
+    it('defaults to an empty string', () => {
+      expect(externalUrl.enterpriseSearchUrl).toEqual('');
+    });
+
+    it('sets the internal enterpriseSearchUrl value', () => {
+      externalUrl.enterpriseSearchUrl = 'http://localhost:3002';
+      expect(externalUrl.enterpriseSearchUrl).toEqual('http://localhost:3002');
+    });
+
+    it('does not allow mutating enterpriseSearchUrl once set', () => {
+      externalUrl.enterpriseSearchUrl = 'hello world';
+      expect(externalUrl.enterpriseSearchUrl).toEqual('http://localhost:3002');
+    });
+  });
+
+  describe('function helpers', () => {
+    it('generates a public Enterprise Search URL', () => {
+      expect(getEnterpriseSearchUrl()).toEqual('http://localhost:3002');
+      expect(getEnterpriseSearchUrl('/login')).toEqual('http://localhost:3002/login');
+    });
+
+    it('generates a public App Search URL', () => {
+      expect(getAppSearchUrl()).toEqual('http://localhost:3002/as');
+      expect(getAppSearchUrl('/path')).toEqual('http://localhost:3002/as/path');
+    });
+
+    it('generates a public Workplace Search URL', () => {
+      expect(getWorkplaceSearchUrl()).toEqual('http://localhost:3002/ws');
+      expect(getWorkplaceSearchUrl('/path')).toEqual('http://localhost:3002/ws/path');
+    });
+  });
+});
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.ts b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.ts
new file mode 100644
index 00000000000000..80b506f31ad614
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/external_url.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * NOTE: The externalUrl obj holds the reference to externalUrl, which should
+ * only ever be updated once on plugin init. We're using a getter and setter
+ * here to ensure it isn't accidentally mutated.
+ *
+ * Someday (8.x+), when our UI is entirely on Kibana and no longer on
+ * Enterprise Search's standalone UI, we can potentially deprecate this helper.
+ */
+export const externalUrl = {
+  _enterpriseSearchUrl: '',
+  get enterpriseSearchUrl() {
+    return this._enterpriseSearchUrl;
+  },
+  set enterpriseSearchUrl(value) {
+    if (this._enterpriseSearchUrl) {
+      // enterpriseSearchUrl is set once on plugin init - we should not mutate it
+      return;
+    }
+    this._enterpriseSearchUrl = value;
+  },
+};
+
+export const getEnterpriseSearchUrl = (path: string = ''): string => {
+  return externalUrl.enterpriseSearchUrl + path;
+};
+export const getAppSearchUrl = (path: string = ''): string => {
+  return getEnterpriseSearchUrl('/as' + path);
+};
+export const getWorkplaceSearchUrl = (path: string = ''): string => {
+  return getEnterpriseSearchUrl('/ws' + path);
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.test.ts b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.test.ts
deleted file mode 100644
index 1092c88cbbc113..00000000000000
--- a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.test.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { ExternalUrl } from './';
-
-describe('Enterprise Search external URL helper', () => {
-  const externalUrl = new ExternalUrl('http://localhost:3002');
-
-  it('exposes a public enterpriseSearchUrl string', () => {
-    expect(externalUrl.enterpriseSearchUrl).toEqual('http://localhost:3002');
-  });
-
-  it('generates a public App Search URL', () => {
-    expect(externalUrl.getAppSearchUrl()).toEqual('http://localhost:3002/as');
-    expect(externalUrl.getAppSearchUrl('/path')).toEqual('http://localhost:3002/as/path');
-  });
-
-  it('generates a public Workplace Search URL', () => {
-    expect(externalUrl.getWorkplaceSearchUrl()).toEqual('http://localhost:3002/ws');
-    expect(externalUrl.getWorkplaceSearchUrl('/path')).toEqual('http://localhost:3002/ws/path');
-  });
-});
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.ts b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.ts
deleted file mode 100644
index 9db48d197f3bc3..00000000000000
--- a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/generate_external_url.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-/**
- * Small helper for generating external public-facing URLs
- * to the legacy/standalone Enterprise Search app
- */
-export interface IExternalUrl {
-  enterpriseSearchUrl?: string;
-  getAppSearchUrl(path?: string): string;
-  getWorkplaceSearchUrl(path?: string): string;
-}
-
-export class ExternalUrl {
-  public enterpriseSearchUrl: string;
-
-  constructor(externalUrl: string) {
-    this.enterpriseSearchUrl = externalUrl;
-
-    this.getAppSearchUrl = this.getAppSearchUrl.bind(this);
-    this.getWorkplaceSearchUrl = this.getWorkplaceSearchUrl.bind(this);
-  }
-
-  private getExternalUrl(path: string): string {
-    return this.enterpriseSearchUrl + path;
-  }
-
-  public getAppSearchUrl(path: string = ''): string {
-    return this.getExternalUrl('/as' + path);
-  }
-
-  public getWorkplaceSearchUrl(path: string = ''): string {
-    return this.getExternalUrl('/ws' + path);
-  }
-}
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/index.ts
index d2d82a43c6dd90..177d8e0535c72a 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/index.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/enterprise_search_url/index.ts
@@ -4,4 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { ExternalUrl, IExternalUrl } from './generate_external_url';
+export {
+  externalUrl,
+  getEnterpriseSearchUrl,
+  getAppSearchUrl,
+  getWorkplaceSearchUrl,
+} from './external_url';
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.test.tsx
index a006c5e3775d52..0ebd59eda5be7f 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.test.tsx
@@ -4,26 +4,26 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { externalUrl } from '../../../shared/enterprise_search_url';
+
 import React from 'react';
 import { shallow } from 'enzyme';
-
 import { EuiButtonEmpty } from '@elastic/eui';
-import { ExternalUrl } from '../../../shared/enterprise_search_url';
 
 import { WorkplaceSearchHeaderActions } from './';
 
 describe('WorkplaceSearchHeaderActions', () => {
-  const externalUrl = new ExternalUrl('http://localhost:3002');
+  it('does not render without an Enterprise Search URL set', () => {
+    const wrapper = shallow(<WorkplaceSearchHeaderActions />);
 
-  it('renders a link to the search application', () => {
-    const wrapper = shallow(<WorkplaceSearchHeaderActions externalUrl={externalUrl} />);
-
-    expect(wrapper.find(EuiButtonEmpty).prop('href')).toEqual('http://localhost:3002/ws/search');
+    expect(wrapper.isEmptyRender()).toBe(true);
   });
 
-  it('does not render without an Enterprise Search host URL set', () => {
-    const wrapper = shallow(<WorkplaceSearchHeaderActions externalUrl={{} as any} />);
+  it('renders a link to the search application', () => {
+    externalUrl.enterpriseSearchUrl = 'http://localhost:3002';
 
-    expect(wrapper.isEmptyRender()).toBe(true);
+    const wrapper = shallow(<WorkplaceSearchHeaderActions />);
+
+    expect(wrapper.find(EuiButtonEmpty).prop('href')).toEqual('http://localhost:3002/ws/search');
   });
 });
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.tsx
index fa32d598f848db..b7da5b4281aa0f 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/kibana_header_actions.tsx
@@ -8,15 +8,10 @@ import React from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiButtonEmpty } from '@elastic/eui';
 
-import { IExternalUrl } from '../../../shared/enterprise_search_url';
+import { externalUrl, getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 
-interface IProps {
-  externalUrl: IExternalUrl;
-}
-
-export const WorkplaceSearchHeaderActions: React.FC<IProps> = ({ externalUrl }) => {
-  const { enterpriseSearchUrl, getWorkplaceSearchUrl } = externalUrl;
-  if (!enterpriseSearchUrl) return null;
+export const WorkplaceSearchHeaderActions: React.FC = () => {
+  if (!externalUrl.enterpriseSearchUrl) return null;
 
   return (
     <EuiButtonEmpty href={getWorkplaceSearchUrl('/search')} target="_blank" iconType="search">
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.test.tsx
index 0e85d8467cff0d..2553284744e4d0 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.test.tsx
@@ -4,7 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import '../../../__mocks__/shallow_usecontext.mock';
+import '../../../__mocks__/enterprise_search_url.mock';
+
 import React from 'react';
 import { shallow } from 'enzyme';
 
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx
index 9fb627ed097910..55727163911128 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx
@@ -3,13 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import React, { useContext } from 'react';
+import React from 'react';
 import { i18n } from '@kbn/i18n';
 
 import { EuiSpacer } from '@elastic/eui';
 
 import { WORKPLACE_SEARCH_PLUGIN } from '../../../../../common/constants';
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 import { SideNav, SideNavLink } from '../../../shared/layout';
 
 import {
@@ -22,10 +22,6 @@ import {
 } from '../../routes';
 
 export const WorkplaceSearchNav: React.FC = () => {
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
-
   // TODO: icons
   return (
     <SideNav product={WORKPLACE_SEARCH_PLUGIN}>
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.test.tsx
index c73eb05ccec16a..2013b2609f33bc 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.test.tsx
@@ -5,7 +5,6 @@
  */
 
 import '../../../../__mocks__/kea.mock';
-import '../../../../__mocks__/shallow_usecontext.mock';
 
 import React from 'react';
 import { shallow } from 'enzyme';
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.tsx
index a80de9fd6ac82c..344b442d9a6781 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/components/shared/product_button/product_button.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { useValues } from 'kea';
 
 import { EuiButton, EuiButtonProps, EuiLinkProps } from '@elastic/eui';
@@ -12,13 +12,10 @@ import { FormattedMessage } from '@kbn/i18n/react';
 
 import { sendTelemetry } from '../../../../shared/telemetry';
 import { HttpLogic } from '../../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../../index';
+import { getWorkplaceSearchUrl } from '../../../../shared/enterprise_search_url';
 
 export const ProductButton: React.FC = () => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const buttonProps = {
     fill: true,
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.test.tsx
index c890adb8ea0438..6be033d7225a87 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.test.tsx
@@ -5,7 +5,7 @@
  */
 
 import '../../../__mocks__/kea.mock';
-import '../../../__mocks__/shallow_usecontext.mock';
+import '../../../__mocks__/enterprise_search_url.mock';
 
 import React from 'react';
 import { shallow } from 'enzyme';
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.tsx
index 79be7ef1cb1587..c1070d57f28567 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_card.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { useValues } from 'kea';
 
 import {
@@ -21,7 +21,7 @@ import {
 
 import { sendTelemetry } from '../../../shared/telemetry';
 import { HttpLogic } from '../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 
 interface IOnboardingCardProps {
   title: React.ReactNode;
@@ -43,9 +43,6 @@ export const OnboardingCard: React.FC<IOnboardingCardProps> = ({
   complete,
 }) => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const onClick = () =>
     sendTelemetry({
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx
index 0f3eee074caefa..37b3340b96a6a5 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx
@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import '../../../__mocks__/shallow_usecontext.mock';
 import './__mocks__/overview_logic.mock';
 import { setMockValues } from './__mocks__';
 
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx
index 079d981533e012..132824833909de 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { useValues } from 'kea';
@@ -24,7 +24,7 @@ import {
 import sharedSourcesIcon from '../../components/shared/assets/share_circle.svg';
 import { sendTelemetry } from '../../../shared/telemetry';
 import { HttpLogic } from '../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 import { ORG_SOURCES_PATH, USERS_PATH, ORG_SETTINGS_PATH } from '../../routes';
 
 import { ContentSection } from '../../components/shared/content_section';
@@ -137,9 +137,6 @@ export const OnboardingSteps: React.FC = () => {
 
 export const OrgNameOnboarding: React.FC = () => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const onClick = () =>
     sendTelemetry({
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx
index 31613098f9fcce..989ff800483f60 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx
@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import '../../../__mocks__/shallow_usecontext.mock';
 import './__mocks__/overview_logic.mock';
 import { setMockValues } from './__mocks__';
 
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx
index dd62e6de7c046d..d1b5228123d947 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
+import React from 'react';
 
 import moment from 'moment';
 import { useValues } from 'kea';
@@ -15,7 +15,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { ContentSection } from '../../components/shared/content_section';
 import { sendTelemetry } from '../../../shared/telemetry';
 import { HttpLogic } from '../../../shared/http';
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 import { SOURCE_DETAILS_PATH, getContentSourcePath } from '../../routes';
 
 import { AppLogic } from '../../app_logic';
@@ -95,9 +95,6 @@ export const RecentActivityItem: React.FC<IFeedActivity> = ({
   sourceId,
 }) => {
   const { http } = useValues(HttpLogic);
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
 
   const onClick = () =>
     sendTelemetry({
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.test.tsx
index edf266231b39ef..013b23d2a9ec0d 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.test.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import '../../../__mocks__/shallow_usecontext.mock';
+import '../../../__mocks__/enterprise_search_url.mock';
 
 import React from 'react';
 import { shallow } from 'enzyme';
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.tsx
index 3e1d285698c0c1..6c4f43b1a3a22b 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/statistic_card.tsx
@@ -4,11 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useContext } from 'react';
-
+import React from 'react';
 import { EuiCard, EuiFlexItem, EuiTitle, EuiTextColor } from '@elastic/eui';
 
-import { KibanaContext, IKibanaContext } from '../../../index';
+import { getWorkplaceSearchUrl } from '../../../shared/enterprise_search_url';
 
 interface IStatisticCardProps {
   title: string;
@@ -17,10 +16,6 @@ interface IStatisticCardProps {
 }
 
 export const StatisticCard: React.FC<IStatisticCardProps> = ({ title, count = 0, actionPath }) => {
-  const {
-    externalUrl: { getWorkplaceSearchUrl },
-  } = useContext(KibanaContext) as IKibanaContext;
-
   const linkProps = actionPath
     ? {
         href: getWorkplaceSearchUrl(actionPath),
diff --git a/x-pack/plugins/enterprise_search/public/plugin.ts b/x-pack/plugins/enterprise_search/public/plugin.ts
index 0e31722cc8bf8c..d870127f297b4d 100644
--- a/x-pack/plugins/enterprise_search/public/plugin.ts
+++ b/x-pack/plugins/enterprise_search/public/plugin.ts
@@ -23,13 +23,13 @@ import {
   WORKPLACE_SEARCH_PLUGIN,
 } from '../common/constants';
 import { IInitialAppData } from '../common/types';
-import { ExternalUrl, IExternalUrl } from './applications/shared/enterprise_search_url';
+import { externalUrl } from './applications/shared/enterprise_search_url';
 
 export interface ClientConfigType {
   host?: string;
 }
 export interface ClientData extends IInitialAppData {
-  externalUrl: IExternalUrl;
+  publicUrl?: string;
   errorConnecting?: boolean;
 }
 
@@ -47,7 +47,6 @@ export class EnterpriseSearchPlugin implements Plugin {
 
   constructor(initializerContext: PluginInitializerContext) {
     this.config = initializerContext.config.get<ClientConfigType>();
-    this.data.externalUrl = new ExternalUrl(this.config.host || '');
   }
 
   public setup(core: CoreSetup, plugins: PluginsSetup) {
@@ -114,7 +113,7 @@ export class EnterpriseSearchPlugin implements Plugin {
           './applications/workplace_search/components/layout'
         );
         params.setHeaderActionMenu((element) =>
-          renderHeaderActions(WorkplaceSearchHeaderActions, element, this.data.externalUrl)
+          renderHeaderActions(WorkplaceSearchHeaderActions, element)
         );
 
         return renderApp(WorkplaceSearch, kibanaDeps, pluginData);
@@ -174,14 +173,14 @@ export class EnterpriseSearchPlugin implements Plugin {
     if (this.hasInitialized) return; // We've already made an initial call
 
     try {
-      const { publicUrl, ...initialData } = await http.get('/api/enterprise_search/config_data');
-      this.data = { ...this.data, ...initialData };
-      if (publicUrl) this.data.externalUrl = new ExternalUrl(publicUrl);
-
+      this.data = await http.get('/api/enterprise_search/config_data');
       this.hasInitialized = true;
+
+      // TODO: This is a temporary workaround to keep the WorkplaceSearchHeaderActions working.
+      // We'll solve this shortly by ensuring the main app store loads before the header actions.
+      externalUrl.enterpriseSearchUrl = this.data.publicUrl || this.config.host;
     } catch {
       this.data.errorConnecting = true;
-      // The plugin will attempt to re-fetch config data on page change
     }
   }
 }

From ace38b4ad445be60bcb2de36be91daca4bd1f3d7 Mon Sep 17 00:00:00 2001
From: spalger <spalger@users.noreply.github.com>
Date: Thu, 24 Sep 2020 14:34:11 -0700
Subject: [PATCH 02/22] skip flaky suite (#78494)

---
 .../reporting_api_integration/reporting_and_security/usage.ts  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts b/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts
index feda5c1386e98d..e2c6e170643736 100644
--- a/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts
+++ b/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts
@@ -21,7 +21,8 @@ export default function ({ getService }: FtrProviderContext) {
   const reportingAPI = getService('reportingAPI');
   const usageAPI = getService('usageAPI');
 
-  describe('Usage', () => {
+  // FLAKY: https://github.com/elastic/kibana/issues/78494
+  describe.skip('Usage', () => {
     before(async () => {
       await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
       await esArchiver.load(OSS_DATA_ARCHIVE_PATH);

From 9d8a389e219de143d7c499fb6fa3b8dd86bc048f Mon Sep 17 00:00:00 2001
From: spalger <spalger@users.noreply.github.com>
Date: Thu, 24 Sep 2020 14:37:02 -0700
Subject: [PATCH 03/22] skip flaky suite (#78288)

---
 x-pack/test/functional/apps/ml/settings/calendar_edit.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/functional/apps/ml/settings/calendar_edit.ts b/x-pack/test/functional/apps/ml/settings/calendar_edit.ts
index f7c8c1f6f85f5b..e738b50a2fe053 100644
--- a/x-pack/test/functional/apps/ml/settings/calendar_edit.ts
+++ b/x-pack/test/functional/apps/ml/settings/calendar_edit.ts
@@ -20,7 +20,8 @@ export default function ({ getService }: FtrProviderContext) {
   const jobConfigs = [createJobConfig('test_calendar_ad_1'), createJobConfig('test_calendar_ad_2')];
   const newJobGroups = ['farequote'];
 
-  describe('calendar edit', function () {
+  // FLAKY: https://github.com/elastic/kibana/issues/78288
+  describe.skip('calendar edit', function () {
     before(async () => {
       await esArchiver.loadIfNeeded('ml/farequote');
       await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

From e53fcbff66508a020f26f0940e56de8b7a4379f5 Mon Sep 17 00:00:00 2001
From: spalger <spalger@users.noreply.github.com>
Date: Thu, 24 Sep 2020 14:42:00 -0700
Subject: [PATCH 04/22] skip flaky suite (#78496)

---
 .../security_solution/cypress/integration/inspect.spec.ts      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/security_solution/cypress/integration/inspect.spec.ts b/x-pack/plugins/security_solution/cypress/integration/inspect.spec.ts
index c19e51c3ada408..8414b4ef8f1a27 100644
--- a/x-pack/plugins/security_solution/cypress/integration/inspect.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/inspect.spec.ts
@@ -21,7 +21,8 @@ import {
 
 import { HOSTS_URL, NETWORK_URL } from '../urls/navigation';
 
-describe('Inspect', () => {
+// FLAKY: https://github.com/elastic/kibana/issues/78496
+describe.skip('Inspect', () => {
   context('Hosts stats and tables', () => {
     before(() => {
       loginAndWaitForPage(HOSTS_URL);

From b3d9809b7d3e085906605b521666d02e3408cad9 Mon Sep 17 00:00:00 2001
From: spalger <spalger@users.noreply.github.com>
Date: Thu, 24 Sep 2020 17:00:36 -0700
Subject: [PATCH 05/22] skip flaky suite (#78512) (#78511) (#78510) (#78509)
 (#78508) (#78507) (#78506) (#78505) (#78504) (#78503) (#78502) (#78501)
 (#78500)

---
 .../integration/ml_conditional_links.spec.ts      | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/security_solution/cypress/integration/ml_conditional_links.spec.ts b/x-pack/plugins/security_solution/cypress/integration/ml_conditional_links.spec.ts
index 7bdc461a7c73d9..3b89163392626e 100644
--- a/x-pack/plugins/security_solution/cypress/integration/ml_conditional_links.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/ml_conditional_links.spec.ts
@@ -24,7 +24,20 @@ import {
   mlNetworkSingleIpNullKqlQuery,
 } from '../urls/ml_conditional_links';
 
-describe('ml conditional links', () => {
+// FLAKY: https://github.com/elastic/kibana/issues/78512
+// FLAKY: https://github.com/elastic/kibana/issues/78511
+// FLAKY: https://github.com/elastic/kibana/issues/78510
+// FLAKY: https://github.com/elastic/kibana/issues/78509
+// FLAKY: https://github.com/elastic/kibana/issues/78508
+// FLAKY: https://github.com/elastic/kibana/issues/78507
+// FLAKY: https://github.com/elastic/kibana/issues/78506
+// FLAKY: https://github.com/elastic/kibana/issues/78505
+// FLAKY: https://github.com/elastic/kibana/issues/78504
+// FLAKY: https://github.com/elastic/kibana/issues/78503
+// FLAKY: https://github.com/elastic/kibana/issues/78502
+// FLAKY: https://github.com/elastic/kibana/issues/78501
+// FLAKY: https://github.com/elastic/kibana/issues/78500
+describe.skip('ml conditional links', () => {
   it('sets the KQL from a single IP with a value for the query', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkSingleIpKqlQuery);
     cy.get(KQL_INPUT)

From d512b5aba4fe7b1848dcebb2ed19c2530e56590b Mon Sep 17 00:00:00 2001
From: "Christiane (Tina) Heiligers" <christiane.heiligers@elastic.co>
Date: Thu, 24 Sep 2020 18:32:55 -0700
Subject: [PATCH 06/22] Adds new elasticsearch client to telemetry plugin
 (#78046)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 src/plugins/data/server/server.api.md         |   1 +
 ...emetry_application_usage_collector.test.ts |  15 +-
 src/plugins/telemetry/server/plugin.ts        |  17 +-
 .../__tests__/get_cluster_info.js             |  39 ---
 .../__tests__/get_cluster_stats.js            |  49 ----
 .../__tests__/get_local_stats.js              | 267 ------------------
 .../get_cluster_info.test.ts                  |  57 ++++
 .../telemetry_collection/get_cluster_info.ts  |  24 +-
 .../get_cluster_stats.test.ts                 |  44 +++
 .../telemetry_collection/get_cluster_stats.ts |  16 +-
 .../get_data_telemetry.test.ts                |  89 +++---
 .../get_data_telemetry/get_data_telemetry.ts  |  60 ++--
 .../get_data_telemetry/index.ts               |   4 +-
 .../server/telemetry_collection/get_kibana.ts |   7 +-
 .../telemetry_collection/get_local_license.ts |  38 ++-
 .../get_local_stats.test.ts                   | 259 +++++++++++++++++
 .../telemetry_collection/get_local_stats.ts   |  25 +-
 .../get_nodes_usage.test.ts                   |  59 ++--
 .../telemetry_collection/get_nodes_usage.ts   |  20 +-
 .../register_collection.ts                    |   5 +-
 .../server/plugin.ts                          |  98 ++++---
 .../server/types.ts                           |  14 +-
 src/plugins/usage_collection/README.md        |   8 +-
 .../server/collector/collector.ts             |   4 +-
 .../server/collector/collector_set.test.ts    |  13 +-
 .../server/collector/collector_set.ts         |  12 +-
 .../usage_collection/server/routes/stats.ts   |  11 +-
 x-pack/plugins/monitoring/server/plugin.ts    |  21 +-
 .../get_all_stats.test.ts                     |   3 +
 .../telemetry_collection/get_all_stats.ts     |   3 +-
 .../get_cluster_uuids.test.ts                 |   6 +-
 .../register_monitoring_collection.ts         |   4 +-
 .../server/plugin.ts                          |  14 +-
 .../__tests__/get_xpack.js                    |  46 ---
 .../get_stats_with_xpack.test.ts              |  91 +++---
 .../get_stats_with_xpack.ts                   |   4 +-
 .../telemetry_collection/get_xpack.test.ts    |  19 ++
 .../server/telemetry_collection/get_xpack.ts  |  13 +-
 38 files changed, 776 insertions(+), 703 deletions(-)
 delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js
 delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js
 delete mode 100644 src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js
 create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts
 create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts
 create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts
 delete mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js
 create mode 100644 x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts

diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md
index f465ece697a700..9cf7234c4a9ffd 100644
--- a/src/plugins/data/server/server.api.md
+++ b/src/plugins/data/server/server.api.md
@@ -44,6 +44,7 @@ import { DeleteDocumentParams } from 'elasticsearch';
 import { DeleteScriptParams } from 'elasticsearch';
 import { DeleteTemplateParams } from 'elasticsearch';
 import { Duration } from 'moment';
+import { ElasticsearchClient as ElasticsearchClient_2 } from 'kibana/server';
 import { Ensure } from '@kbn/utility-types';
 import { EnvironmentMode } from '@kbn/config';
 import { ErrorToastOptions } from 'src/core/public/notifications';
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
index 709736a37d8026..23a77c2d4c288a 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
@@ -17,7 +17,11 @@
  * under the License.
  */
 
-import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
+import {
+  savedObjectsRepositoryMock,
+  loggingSystemMock,
+  elasticsearchServiceMock,
+} from '../../../../../core/server/mocks';
 import {
   CollectorOptions,
   createUsageCollectionSetupMock,
@@ -50,6 +54,7 @@ describe('telemetry_application_usage', () => {
   const getUsageCollector = jest.fn();
   const registerType = jest.fn();
   const callCluster = jest.fn();
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
 
   beforeAll(() =>
     registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
@@ -62,7 +67,7 @@ describe('telemetry_application_usage', () => {
 
   test('if no savedObjectClient initialised, return undefined', async () => {
     expect(collector.isReady()).toBe(false);
-    expect(await collector.fetch(callCluster)).toBeUndefined();
+    expect(await collector.fetch(callCluster, esClient)).toBeUndefined();
     jest.runTimersToTime(ROLL_INDICES_START);
   });
 
@@ -80,7 +85,7 @@ describe('telemetry_application_usage', () => {
     jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
 
     expect(collector.isReady()).toBe(true);
-    expect(await collector.fetch(callCluster)).toStrictEqual({});
+    expect(await collector.fetch(callCluster, esClient)).toStrictEqual({});
     expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
   });
 
@@ -137,7 +142,7 @@ describe('telemetry_application_usage', () => {
 
     jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
 
-    expect(await collector.fetch(callCluster)).toStrictEqual({
+    expect(await collector.fetch(callCluster, esClient)).toStrictEqual({
       appId: {
         clicks_total: total + 1 + 10,
         clicks_7_days: total + 1,
@@ -197,7 +202,7 @@ describe('telemetry_application_usage', () => {
 
     getUsageCollector.mockImplementation(() => savedObjectClient);
 
-    expect(await collector.fetch(callCluster)).toStrictEqual({
+    expect(await collector.fetch(callCluster, esClient)).toStrictEqual({
       appId: {
         clicks_total: 1,
         clicks_7_days: 0,
diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts
index 005c5f96d98d03..dfbbe3355e69c9 100644
--- a/src/plugins/telemetry/server/plugin.ts
+++ b/src/plugins/telemetry/server/plugin.ts
@@ -34,6 +34,7 @@ import {
   SavedObjectsClient,
   Plugin,
   Logger,
+  IClusterClient,
 } from '../../../core/server';
 import { registerRoutes } from './routes';
 import { registerCollection } from './telemetry_collection';
@@ -83,6 +84,7 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
   private readonly fetcherTask: FetcherTask;
   private savedObjectsClient?: ISavedObjectsRepository;
   private uiSettingsClient?: IUiSettingsClient;
+  private elasticsearchClient?: IClusterClient;
 
   constructor(initializerContext: PluginInitializerContext<TelemetryConfigType>) {
     this.logger = initializerContext.logger.get();
@@ -102,8 +104,11 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
     const currentKibanaVersion = this.currentKibanaVersion;
     const config$ = this.config$;
     const isDev = this.isDev;
-
-    registerCollection(telemetryCollectionManager, elasticsearch.legacy.client);
+    registerCollection(
+      telemetryCollectionManager,
+      elasticsearch.legacy.client,
+      () => this.elasticsearchClient
+    );
     const router = http.createRouter();
 
     registerRoutes({
@@ -126,14 +131,12 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
     };
   }
 
-  public async start(
-    core: CoreStart,
-    { telemetryCollectionManager }: TelemetryPluginsDepsStart
-  ): Promise<TelemetryPluginStart> {
-    const { savedObjects, uiSettings } = core;
+  public async start(core: CoreStart, { telemetryCollectionManager }: TelemetryPluginsDepsStart) {
+    const { savedObjects, uiSettings, elasticsearch } = core;
     this.savedObjectsClient = savedObjects.createInternalRepository();
     const savedObjectsClient = new SavedObjectsClient(this.savedObjectsClient);
     this.uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
+    this.elasticsearchClient = elasticsearch.client;
 
     try {
       await handleOldSettings(savedObjectsClient, this.uiSettingsClient);
diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js
deleted file mode 100644
index fe83b76cd11583..00000000000000
--- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_info.js
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import expect from '@kbn/expect';
-import sinon from 'sinon';
-
-import { getClusterInfo } from '../get_cluster_info';
-
-export function mockGetClusterInfo(callCluster, clusterInfo, req) {
-  callCluster.withArgs(req, 'info').returns(clusterInfo);
-  callCluster.withArgs('info').returns(clusterInfo);
-}
-
-describe('get_cluster_info', () => {
-  it('uses callCluster to get info API', () => {
-    const callCluster = sinon.stub();
-    const response = Promise.resolve({});
-
-    mockGetClusterInfo(callCluster, response);
-
-    expect(getClusterInfo(callCluster)).to.be(response);
-  });
-});
diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js
deleted file mode 100644
index d1354608385f64..00000000000000
--- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_cluster_stats.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import expect from '@kbn/expect';
-import sinon from 'sinon';
-
-import { TIMEOUT } from '../constants';
-import { getClusterStats } from '../get_cluster_stats';
-
-export function mockGetClusterStats(callCluster, clusterStats, req) {
-  callCluster
-    .withArgs(req, 'cluster.stats', {
-      timeout: TIMEOUT,
-    })
-    .returns(clusterStats);
-
-  callCluster
-    .withArgs('cluster.stats', {
-      timeout: TIMEOUT,
-    })
-    .returns(clusterStats);
-}
-
-describe.skip('get_cluster_stats', () => {
-  it('uses callCluster to get cluster.stats API', async () => {
-    const callCluster = sinon.stub();
-    const response = Promise.resolve({});
-
-    mockGetClusterStats(callCluster, response);
-
-    expect(getClusterStats(callCluster)).to.be(response);
-  });
-});
diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js
deleted file mode 100644
index 8541745faea3b6..00000000000000
--- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import expect from '@kbn/expect';
-import sinon from 'sinon';
-import { merge, omit } from 'lodash';
-
-import { TIMEOUT } from '../constants';
-import { mockGetClusterInfo } from './get_cluster_info';
-import { mockGetClusterStats } from './get_cluster_stats';
-
-import { getLocalStats, handleLocalStats } from '../get_local_stats';
-
-const mockUsageCollection = (kibanaUsage = {}) => ({
-  bulkFetch: () => kibanaUsage,
-  toObject: (data) => data,
-});
-
-const getMockServer = (getCluster = sinon.stub()) => ({
-  log(tags, message) {
-    console.log({ tags, message });
-  },
-  config() {
-    return {
-      get(item) {
-        switch (item) {
-          case 'pkg.version':
-            return '8675309-snapshot';
-          default:
-            throw Error(`unexpected config.get('${item}') received.`);
-        }
-      },
-    };
-  },
-  plugins: {
-    elasticsearch: { getCluster },
-  },
-});
-function mockGetNodesUsage(callCluster, nodesUsage, req) {
-  callCluster
-    .withArgs(
-      req,
-      {
-        method: 'GET',
-        path: '/_nodes/usage',
-        query: {
-          timeout: TIMEOUT,
-        },
-      },
-      'transport.request'
-    )
-    .returns(nodesUsage);
-}
-
-function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) {
-  mockGetClusterInfo(callCluster, clusterInfo, req);
-  mockGetClusterStats(callCluster, clusterStats, req);
-  mockGetNodesUsage(callCluster, nodesUsage, req);
-}
-
-describe('get_local_stats', () => {
-  const clusterUuid = 'abc123';
-  const clusterName = 'my-cool-cluster';
-  const version = '2.3.4';
-  const clusterInfo = {
-    cluster_uuid: clusterUuid,
-    cluster_name: clusterName,
-    version: {
-      number: version,
-    },
-  };
-  const nodesUsage = [
-    {
-      node_id: 'some_node_id',
-      timestamp: 1588617023177,
-      since: 1588616945163,
-      rest_actions: {
-        nodes_usage_action: 1,
-        create_index_action: 1,
-        document_get_action: 1,
-        search_action: 19,
-        nodes_info_action: 36,
-      },
-      aggregations: {
-        terms: {
-          bytes: 2,
-        },
-        scripted_metric: {
-          other: 7,
-        },
-      },
-    },
-  ];
-  const clusterStats = {
-    _nodes: { failed: 123 },
-    cluster_name: 'real-cool',
-    indices: { totally: 456 },
-    nodes: { yup: 'abc' },
-    random: 123,
-  };
-
-  const kibana = {
-    kibana: {
-      great: 'googlymoogly',
-      versions: [{ version: '8675309', count: 1 }],
-    },
-    kibana_stats: {
-      os: {
-        platform: 'rocky',
-        platformRelease: 'iv',
-      },
-    },
-    localization: {
-      locale: 'en',
-      labelsCount: 0,
-      integrities: {},
-    },
-    sun: { chances: 5 },
-    clouds: { chances: 95 },
-    rain: { chances: 2 },
-    snow: { chances: 0 },
-  };
-
-  const clusterStatsWithNodesUsage = {
-    ...clusterStats,
-    nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
-  };
-  const combinedStatsResult = {
-    collection: 'local',
-    cluster_uuid: clusterUuid,
-    cluster_name: clusterName,
-    version,
-    cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
-    stack_stats: {
-      kibana: {
-        great: 'googlymoogly',
-        count: 1,
-        indices: 1,
-        os: {
-          platforms: [{ platform: 'rocky', count: 1 }],
-          platformReleases: [{ platformRelease: 'iv', count: 1 }],
-        },
-        versions: [{ version: '8675309', count: 1 }],
-        plugins: {
-          localization: {
-            locale: 'en',
-            labelsCount: 0,
-            integrities: {},
-          },
-          sun: { chances: 5 },
-          clouds: { chances: 95 },
-          rain: { chances: 2 },
-          snow: { chances: 0 },
-        },
-      },
-    },
-  };
-
-  const context = {
-    logger: console,
-    version: '8.0.0',
-  };
-
-  describe('handleLocalStats', () => {
-    it('returns expected object without xpack and kibana data', () => {
-      const result = handleLocalStats(
-        clusterInfo,
-        clusterStatsWithNodesUsage,
-        void 0,
-        void 0,
-        context
-      );
-      expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
-      expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
-      expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
-      expect(result.version).to.be('2.3.4');
-      expect(result.collection).to.be('local');
-      expect(result.license).to.be(undefined);
-      expect(result.stack_stats).to.eql({ kibana: undefined, data: undefined });
-    });
-
-    it('returns expected object with xpack', () => {
-      const result = handleLocalStats(
-        clusterInfo,
-        clusterStatsWithNodesUsage,
-        void 0,
-        void 0,
-        context
-      );
-      const { stack_stats: stack, ...cluster } = result;
-      expect(cluster.collection).to.be(combinedStatsResult.collection);
-      expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
-      expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name);
-      expect(stack.kibana).to.be(undefined); // not mocked for this test
-      expect(stack.data).to.be(undefined); // not mocked for this test
-
-      expect(cluster.version).to.eql(combinedStatsResult.version);
-      expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
-      expect(cluster.license).to.eql(combinedStatsResult.license);
-      expect(stack.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
-    });
-  });
-
-  describe.skip('getLocalStats', () => {
-    it('returns expected object without xpack data when X-Pack fails to respond', async () => {
-      const callClusterUsageFailed = sinon.stub();
-      const usageCollection = mockUsageCollection();
-      mockGetLocalStats(
-        callClusterUsageFailed,
-        Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats),
-        Promise.resolve(nodesUsage)
-      );
-      const result = await getLocalStats([], {
-        server: getMockServer(),
-        callCluster: callClusterUsageFailed,
-        usageCollection,
-      });
-      expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
-      expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
-      expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
-      expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes);
-      expect(result.version).to.be('2.3.4');
-      expect(result.collection).to.be('local');
-
-      // license and xpack usage info come from the same cluster call
-      expect(result.license).to.be(undefined);
-      expect(result.stack_stats.xpack).to.be(undefined);
-    });
-
-    it('returns expected object with xpack and kibana data', async () => {
-      const callCluster = sinon.stub();
-      const usageCollection = mockUsageCollection(kibana);
-      mockGetLocalStats(
-        callCluster,
-        Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats),
-        Promise.resolve(nodesUsage)
-      );
-
-      const result = await getLocalStats([], {
-        server: getMockServer(callCluster),
-        usageCollection,
-        callCluster,
-      });
-
-      expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
-      expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana);
-    });
-  });
-});
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts
new file mode 100644
index 00000000000000..459b18d252e171
--- /dev/null
+++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.test.ts
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+import { getClusterInfo } from './get_cluster_info';
+
+export function mockGetClusterInfo(clusterInfo: any) {
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+  esClient.info
+    // @ts-ignore we only care about the response body
+    .mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      {
+        body: { ...clusterInfo },
+      }
+    );
+  return esClient;
+}
+
+describe('get_cluster_info using the elasticsearch client', () => {
+  it('uses the esClient to get info API', async () => {
+    const clusterInfo = {
+      cluster_uuid: '1234',
+      cluster_name: 'testCluster',
+      version: {
+        number: '7.9.2',
+        build_flavor: 'default',
+        build_type: 'docker',
+        build_hash: 'b5ca9c58fb664ca8bf',
+        build_date: '2020-07-21T16:40:44.668009Z',
+        build_snapshot: false,
+        lucene_version: '8.5.1',
+        minimum_wire_compatibility_version: '6.8.0',
+        minimum_index_compatibility_version: '6.0.0-beta1',
+      },
+    };
+    const esClient = mockGetClusterInfo(clusterInfo);
+
+    expect(await getClusterInfo(esClient)).toStrictEqual(clusterInfo);
+  });
+});
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts
index 4a33356ee97614..407f3325c3a9f5 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { LegacyAPICaller } from 'kibana/server';
+import { ElasticsearchClient } from 'src/core/server';
 
 // This can be removed when the ES client improves the types
 export interface ESClusterInfo {
@@ -25,24 +25,24 @@ export interface ESClusterInfo {
   cluster_name: string;
   version: {
     number: string;
-    build_flavor: string;
-    build_type: string;
-    build_hash: string;
-    build_date: string;
+    build_flavor?: string;
+    build_type?: string;
+    build_hash?: string;
+    build_date?: string;
     build_snapshot?: boolean;
-    lucene_version: string;
-    minimum_wire_compatibility_version: string;
-    minimum_index_compatibility_version: string;
+    lucene_version?: string;
+    minimum_wire_compatibility_version?: string;
+    minimum_index_compatibility_version?: string;
   };
 }
-
 /**
  * Get the cluster info from the connected cluster.
  *
  * This is the equivalent to GET /
  *
- * @param {function} callCluster The callWithInternalUser handler (exposed for testing)
+ * @param {function} esClient The asInternalUser handler (exposed for testing)
  */
-export function getClusterInfo(callCluster: LegacyAPICaller) {
-  return callCluster<ESClusterInfo>('info');
+export async function getClusterInfo(esClient: ElasticsearchClient) {
+  const { body } = await esClient.info<ESClusterInfo>();
+  return body;
 }
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts
new file mode 100644
index 00000000000000..81551c0c4d93db
--- /dev/null
+++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.test.ts
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+import { getClusterStats } from './get_cluster_stats';
+import { TIMEOUT } from './constants';
+
+export function mockGetClusterStats(clusterStats: any) {
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+  esClient.cluster.stats.mockResolvedValue(clusterStats);
+  return esClient;
+}
+
+describe('get_cluster_stats', () => {
+  it('uses the esClient to get the response from the `cluster.stats` API', async () => {
+    const response = Promise.resolve({ body: { cluster_uuid: '1234' } });
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    esClient.cluster.stats.mockImplementationOnce(
+      // @ts-ignore the method only cares about the response body
+      async (_params = { timeout: TIMEOUT }) => {
+        return response;
+      }
+    );
+    const result = getClusterStats(esClient);
+    expect(esClient.cluster.stats).toHaveBeenCalledWith({ timeout: TIMEOUT });
+    expect(result).toStrictEqual(response);
+  });
+});
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts
index d7c0110a99c6fd..d2a64e48786799 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_cluster_stats.ts
@@ -18,23 +18,23 @@
  */
 
 import { ClusterDetailsGetter } from 'src/plugins/telemetry_collection_manager/server';
-import { LegacyAPICaller } from 'kibana/server';
+import { ElasticsearchClient } from 'src/core/server';
 import { TIMEOUT } from './constants';
 /**
  * Get the cluster stats from the connected cluster.
  *
  * This is the equivalent to GET /_cluster/stats?timeout=30s.
  */
-export async function getClusterStats(callCluster: LegacyAPICaller) {
-  return await callCluster('cluster.stats', {
-    timeout: TIMEOUT,
-  });
+export async function getClusterStats(esClient: ElasticsearchClient) {
+  const { body } = await esClient.cluster.stats({ timeout: TIMEOUT });
+  return body;
 }
 
 /**
  * Get the cluster uuids from the connected cluster.
  */
-export const getClusterUuids: ClusterDetailsGetter = async ({ callCluster }) => {
-  const result = await getClusterStats(callCluster);
-  return [{ clusterUuid: result.cluster_uuid }];
+export const getClusterUuids: ClusterDetailsGetter = async ({ esClient }) => {
+  const { body } = await esClient.cluster.stats({ timeout: TIMEOUT });
+
+  return [{ clusterUuid: body.cluster_uuid }];
 };
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts
index dee718decdc1f7..bb5eb7f6b726da 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts
@@ -19,6 +19,7 @@
 
 import { buildDataTelemetryPayload, getDataTelemetry } from './get_data_telemetry';
 import { DATA_DATASETS_INDEX_PATTERNS, DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants';
+import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks';
 
 describe('get_data_telemetry', () => {
   describe('DATA_DATASETS_INDEX_PATTERNS', () => {
@@ -195,13 +196,15 @@ describe('get_data_telemetry', () => {
 
   describe('getDataTelemetry', () => {
     test('it returns the base payload (all 0s) because no indices are found', async () => {
-      const callCluster = mockCallCluster();
-      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]);
+      const esClient = mockEsClient();
+      await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]);
+      expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1);
+      expect(esClient.indices.stats).toHaveBeenCalledTimes(1);
     });
 
     test('can only see the index mappings, but not the stats', async () => {
-      const callCluster = mockCallCluster(['filebeat-12314']);
-      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+      const esClient = mockEsClient(['filebeat-12314']);
+      await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([
         {
           pattern_name: 'filebeat',
           shipper: 'filebeat',
@@ -209,10 +212,12 @@ describe('get_data_telemetry', () => {
           ecs_index_count: 0,
         },
       ]);
+      expect(esClient.indices.getMapping).toHaveBeenCalledTimes(1);
+      expect(esClient.indices.stats).toHaveBeenCalledTimes(1);
     });
 
     test('can see the mappings and the stats', async () => {
-      const callCluster = mockCallCluster(
+      const esClient = mockEsClient(
         ['filebeat-12314'],
         { isECS: true },
         {
@@ -221,7 +226,7 @@ describe('get_data_telemetry', () => {
           },
         }
       );
-      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+      await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([
         {
           pattern_name: 'filebeat',
           shipper: 'filebeat',
@@ -234,7 +239,7 @@ describe('get_data_telemetry', () => {
     });
 
     test('find an index that does not match any index pattern but has mappings metadata', async () => {
-      const callCluster = mockCallCluster(
+      const esClient = mockEsClient(
         ['cannot_match_anything'],
         { isECS: true, dataStreamType: 'traces', shipper: 'my-beat' },
         {
@@ -245,7 +250,7 @@ describe('get_data_telemetry', () => {
           },
         }
       );
-      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([
+      await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([
         {
           data_stream: { dataset: undefined, type: 'traces' },
           shipper: 'my-beat',
@@ -258,45 +263,51 @@ describe('get_data_telemetry', () => {
     });
 
     test('return empty array when there is an error', async () => {
-      const callCluster = jest.fn().mockRejectedValue(new Error('Something went terribly wrong'));
-      await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]);
+      const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      esClient.indices.getMapping.mockRejectedValue(new Error('Something went terribly wrong'));
+      esClient.indices.stats.mockRejectedValue(new Error('Something went terribly wrong'));
+      await expect(getDataTelemetry(esClient)).resolves.toStrictEqual([]);
     });
   });
 });
-
-function mockCallCluster(
-  indicesMappings: string[] = [],
+function mockEsClient(
+  indicesMappings: string[] = [], // an array of `indices` to get mappings from.
   { isECS = false, dataStreamDataset = '', dataStreamType = '', shipper = '' } = {},
   indexStats: any = {}
 ) {
-  return jest.fn().mockImplementation(async (method: string, opts: any) => {
-    if (method === 'indices.getMapping') {
-      return Object.fromEntries(
-        indicesMappings.map((index) => [
-          index,
-          {
-            mappings: {
-              ...(shipper && { _meta: { beat: shipper } }),
-              properties: {
-                ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }),
-                ...((dataStreamType || dataStreamDataset) && {
-                  data_stream: {
-                    properties: {
-                      ...(dataStreamDataset && {
-                        dataset: { type: 'constant_keyword', value: dataStreamDataset },
-                      }),
-                      ...(dataStreamType && {
-                        type: { type: 'constant_keyword', value: dataStreamType },
-                      }),
-                    },
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+  // @ts-ignore
+  esClient.indices.getMapping.mockImplementationOnce(async () => {
+    const body = Object.fromEntries(
+      indicesMappings.map((index) => [
+        index,
+        {
+          mappings: {
+            ...(shipper && { _meta: { beat: shipper } }),
+            properties: {
+              ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }),
+              ...((dataStreamType || dataStreamDataset) && {
+                data_stream: {
+                  properties: {
+                    ...(dataStreamDataset && {
+                      dataset: { type: 'constant_keyword', value: dataStreamDataset },
+                    }),
+                    ...(dataStreamType && {
+                      type: { type: 'constant_keyword', value: dataStreamType },
+                    }),
                   },
-                }),
-              },
+                },
+              }),
             },
           },
-        ])
-      );
-    }
-    return indexStats;
+        },
+      ])
+    );
+    return { body };
+  });
+  // @ts-ignore
+  esClient.indices.stats.mockImplementationOnce(async () => {
+    return { body: indexStats };
   });
+  return esClient;
 }
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts
index f4734dde251cc8..67769793cbfdf8 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts
@@ -16,8 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import { ElasticsearchClient } from 'src/core/server';
 
-import { LegacyAPICaller } from 'kibana/server';
 import {
   DATA_DATASETS_INDEX_PATTERNS_UNIQUE,
   DataPatternName,
@@ -224,42 +224,50 @@ interface IndexMappings {
   };
 }
 
-export async function getDataTelemetry(callCluster: LegacyAPICaller) {
+export async function getDataTelemetry(esClient: ElasticsearchClient) {
   try {
     const index = [
       ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern),
       '*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}`
     ];
-    const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([
+    const indexMappingsParams: { index: string; filter_path: string[] } = {
       // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value
-      callCluster('indices.getMapping', {
-        index: '*', // Request all indices because filter_path already filters out the indices without any of those fields
-        filterPath: [
-          // _meta.beat tells the shipper
-          '*.mappings._meta.beat',
-          // _meta.package.name tells the Ingest Manager's package
-          '*.mappings._meta.package.name',
-          // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it
-          '*.mappings._meta.managed_by',
-          // Does it have `ecs.version` in the mappings? => It follows the ECS conventions
-          '*.mappings.properties.ecs.properties.version.type',
+      index: '*', // Request all indices because filter_path already filters out the indices without any of those fields
+      filter_path: [
+        // _meta.beat tells the shipper
+        '*.mappings._meta.beat',
+        // _meta.package.name tells the Ingest Manager's package
+        '*.mappings._meta.package.name',
+        // _meta.managed_by is usually populated by Ingest Manager for the UI to identify it
+        '*.mappings._meta.managed_by',
+        // Does it have `ecs.version` in the mappings? => It follows the ECS conventions
+        '*.mappings.properties.ecs.properties.version.type',
 
-          // If `data_stream.type` is a `constant_keyword`, it can be reported as a type
-          '*.mappings.properties.data_stream.properties.type.value',
-          // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset
-          '*.mappings.properties.data_stream.properties.dataset.value',
-        ],
-      }),
+        // If `data_stream.type` is a `constant_keyword`, it can be reported as a type
+        '*.mappings.properties.data_stream.properties.type.value',
+        // If `data_stream.dataset` is a `constant_keyword`, it can be reported as the dataset
+        '*.mappings.properties.data_stream.properties.dataset.value',
+      ],
+    };
+    const indicesStatsParams: {
+      index: string | string[] | undefined;
+      level: 'cluster' | 'indices' | 'shards' | undefined;
+      metric: string[];
+      filter_path: string[];
+    } = {
       // GET <index>/_stats/docs,store?level=indices&filter_path=indices.*.total
-      callCluster<IndexStats>('indices.stats', {
-        index,
-        level: 'indices',
-        metric: ['docs', 'store'],
-        filterPath: ['indices.*.total'],
-      }),
+      index,
+      level: 'indices',
+      metric: ['docs', 'store'],
+      filter_path: ['indices.*.total'],
+    };
+    const [{ body: indexMappings }, { body: indexStats }] = await Promise.all([
+      esClient.indices.getMapping<IndexMappings>(indexMappingsParams),
+      esClient.indices.stats<IndexStats>(indicesStatsParams),
     ]);
 
     const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices });
+
     const indices = indexNames.map((name) => {
       const baseIndexInfo = {
         name,
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts
index d056d1c9f299f3..0e2ab98a24cbaa 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts
@@ -20,8 +20,8 @@
 export { DATA_TELEMETRY_ID } from './constants';
 
 export {
-  DataTelemetryIndex,
-  DataTelemetryPayload,
   getDataTelemetry,
   buildDataTelemetryPayload,
+  DataTelemetryPayload,
+  DataTelemetryIndex,
 } from './get_data_telemetry';
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts
index 5d27774a630a59..0ef9815a4eadb1 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts
@@ -21,6 +21,7 @@ import { omit } from 'lodash';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { LegacyAPICaller } from 'kibana/server';
 import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server';
+import { ElasticsearchClient } from 'src/core/server';
 
 export interface KibanaUsageStats {
   kibana: {
@@ -48,7 +49,6 @@ export function handleKibanaStats(
     logger.warn('No Kibana stats returned from usage collectors');
     return;
   }
-
   const { kibana, kibana_stats: kibanaStats, ...plugins } = response;
 
   const os = {
@@ -83,8 +83,9 @@ export function handleKibanaStats(
 
 export async function getKibana(
   usageCollection: UsageCollectionSetup,
-  callWithInternalUser: LegacyAPICaller
+  callWithInternalUser: LegacyAPICaller,
+  asInternalUser: ElasticsearchClient
 ): Promise<KibanaUsageStats> {
-  const usage = await usageCollection.bulkFetch(callWithInternalUser);
+  const usage = await usageCollection.bulkFetch(callWithInternalUser, asInternalUser);
   return usageCollection.toObject(usage);
 }
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts
index d41904c6d8e0e8..879416cda62fc4 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_local_license.ts
@@ -17,44 +17,41 @@
  * under the License.
  */
 
-import { LegacyAPICaller } from 'kibana/server';
 import { ESLicense, LicenseGetter } from 'src/plugins/telemetry_collection_manager/server';
+import { ElasticsearchClient } from 'src/core/server';
 
 let cachedLicense: ESLicense | undefined;
 
-function fetchLicense(callCluster: LegacyAPICaller, local: boolean) {
-  return callCluster<{ license: ESLicense }>('transport.request', {
-    method: 'GET',
-    path: '/_license',
-    query: {
-      local,
-      // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license.
-      accept_enterprise: 'true',
-    },
+async function fetchLicense(esClient: ElasticsearchClient, local: boolean) {
+  const { body } = await esClient.license.get({
+    local,
+    // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license.
+    accept_enterprise: true,
   });
+  return body;
 }
-
 /**
  * Get the cluster's license from the connected node.
  *
- * This is the equivalent of GET /_license?local=true .
+ * This is the equivalent of GET /_license?local=true&accept_enterprise=true.
  *
  * Like any X-Pack related API, X-Pack must installed for this to work.
+ *
+ * In OSS we'll get a 400 response using the new elasticsearch client.
  */
-async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) {
-  // Fetching the local license is cheaper than getting it from the master and good enough
-  const { license } = await fetchLicense(callCluster, true).catch(async (err) => {
+async function getLicenseFromLocalOrMaster(esClient: ElasticsearchClient) {
+  // Fetching the local license is cheaper than getting it from the master node and good enough
+  const { license } = await fetchLicense(esClient, true).catch(async (err) => {
     if (cachedLicense) {
       try {
         // Fallback to the master node's license info
-        const response = await fetchLicense(callCluster, false);
+        const response = await fetchLicense(esClient, false);
         return response;
       } catch (masterError) {
-        if (masterError.statusCode === 404) {
+        if ([400, 404].includes(masterError.statusCode)) {
           // If the master node does not have a license, we can assume there is no license
           cachedLicense = undefined;
         } else {
-          // Any other errors from the master node, throw and do not send any telemetry
           throw err;
         }
       }
@@ -68,9 +65,8 @@ async function getLicenseFromLocalOrMaster(callCluster: LegacyAPICaller) {
   return license;
 }
 
-export const getLocalLicense: LicenseGetter = async (clustersDetails, { callCluster }) => {
-  const license = await getLicenseFromLocalOrMaster(callCluster);
-
+export const getLocalLicense: LicenseGetter = async (clustersDetails, { esClient }) => {
+  const license = await getLicenseFromLocalOrMaster(esClient);
   // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case.
   return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {});
 };
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts
new file mode 100644
index 00000000000000..0c8b0b249f7d11
--- /dev/null
+++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { merge, omit } from 'lodash';
+
+import { getLocalStats, handleLocalStats } from './get_local_stats';
+import { usageCollectionPluginMock } from '../../../usage_collection/server/mocks';
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+
+function mockUsageCollection(kibanaUsage = {}) {
+  const usageCollection = usageCollectionPluginMock.createSetupContract();
+  usageCollection.bulkFetch = jest.fn().mockResolvedValue(kibanaUsage);
+  usageCollection.toObject = jest.fn().mockImplementation((data: any) => data);
+  return usageCollection;
+}
+// set up successful call mocks for info, cluster stats, nodes usage and data telemetry
+function mockGetLocalStats(clusterInfo: any, clusterStats: any) {
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+  esClient.info
+    // @ts-ignore we only care about the response body
+    .mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      {
+        body: { ...clusterInfo },
+      }
+    );
+  esClient.cluster.stats
+    // @ts-ignore we only care about the response body
+    .mockResolvedValue({ body: { ...clusterStats } });
+  esClient.nodes.usage.mockResolvedValue(
+    // @ts-ignore we only care about the response body
+    {
+      body: {
+        cluster_name: 'testCluster',
+        nodes: {
+          some_node_id: {
+            timestamp: 1588617023177,
+            since: 1588616945163,
+            rest_actions: {
+              nodes_usage_action: 1,
+              create_index_action: 1,
+              document_get_action: 1,
+              search_action: 19,
+              nodes_info_action: 36,
+            },
+            aggregations: {
+              terms: {
+                bytes: 2,
+              },
+              scripted_metric: {
+                other: 7,
+              },
+            },
+          },
+        },
+      },
+    }
+  );
+  // @ts-ignore we only care about the response body
+  esClient.indices.getMapping.mockResolvedValue({ body: { mappings: {} } });
+  // @ts-ignore we only care about the response body
+  esClient.indices.stats.mockResolvedValue({ body: { indices: {} } });
+  return esClient;
+}
+
+describe('get_local_stats', () => {
+  const clusterUuid = 'abc123';
+  const clusterName = 'my-cool-cluster';
+  const version = '2.3.4';
+  const clusterInfo = {
+    cluster_uuid: clusterUuid,
+    cluster_name: clusterName,
+    version: { number: version },
+  };
+  const nodesUsage = [
+    {
+      node_id: 'some_node_id',
+      timestamp: 1588617023177,
+      since: 1588616945163,
+      rest_actions: {
+        nodes_usage_action: 1,
+        create_index_action: 1,
+        document_get_action: 1,
+        search_action: 19,
+        nodes_info_action: 36,
+      },
+      aggregations: {
+        terms: {
+          bytes: 2,
+        },
+        scripted_metric: {
+          other: 7,
+        },
+      },
+    },
+  ];
+  const clusterStats = {
+    _nodes: { failed: 123 },
+    cluster_name: 'real-cool',
+    indices: { totally: 456 },
+    nodes: { yup: 'abc' },
+    random: 123,
+  };
+
+  const kibana = {
+    kibana: {
+      great: 'googlymoogly',
+      versions: [{ version: '8675309', count: 1 }],
+    },
+    kibana_stats: {
+      os: {
+        platform: 'rocky',
+        platformRelease: 'iv',
+      },
+    },
+    localization: {
+      locale: 'en',
+      labelsCount: 0,
+      integrities: {},
+    },
+    sun: { chances: 5 },
+    clouds: { chances: 95 },
+    rain: { chances: 2 },
+    snow: { chances: 0 },
+  };
+
+  const clusterStatsWithNodesUsage = {
+    ...clusterStats,
+    nodes: merge(clusterStats.nodes, { usage: { nodes: nodesUsage } }),
+  };
+
+  const combinedStatsResult = {
+    collection: 'local',
+    cluster_uuid: clusterUuid,
+    cluster_name: clusterName,
+    version,
+    cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
+    stack_stats: {
+      kibana: {
+        great: 'googlymoogly',
+        count: 1,
+        indices: 1,
+        os: {
+          platforms: [{ platform: 'rocky', count: 1 }],
+          platformReleases: [{ platformRelease: 'iv', count: 1 }],
+        },
+        versions: [{ version: '8675309', count: 1 }],
+        plugins: {
+          localization: {
+            locale: 'en',
+            labelsCount: 0,
+            integrities: {},
+          },
+          sun: { chances: 5 },
+          clouds: { chances: 95 },
+          rain: { chances: 2 },
+          snow: { chances: 0 },
+        },
+      },
+    },
+  };
+
+  const context = {
+    logger: console,
+    version: '8.0.0',
+  };
+
+  describe('handleLocalStats', () => {
+    it('returns expected object without xpack or kibana data', () => {
+      const result = handleLocalStats(
+        clusterInfo,
+        clusterStatsWithNodesUsage,
+        void 0,
+        void 0,
+        context
+      );
+      expect(result.cluster_uuid).toStrictEqual(combinedStatsResult.cluster_uuid);
+      expect(result.cluster_name).toStrictEqual(combinedStatsResult.cluster_name);
+      expect(result.cluster_stats).toStrictEqual(combinedStatsResult.cluster_stats);
+      expect(result.version).toEqual('2.3.4');
+      expect(result.collection).toEqual('local');
+      expect(Object.keys(result)).not.toContain('license');
+      expect(result.stack_stats).toEqual({ kibana: undefined, data: undefined });
+    });
+
+    it('returns expected object with xpack', () => {
+      const result = handleLocalStats(
+        clusterInfo,
+        clusterStatsWithNodesUsage,
+        void 0,
+        void 0,
+        context
+      );
+
+      const { stack_stats: stack, ...cluster } = result;
+      expect(cluster.collection).toBe(combinedStatsResult.collection);
+      expect(cluster.cluster_uuid).toBe(combinedStatsResult.cluster_uuid);
+      expect(cluster.cluster_name).toBe(combinedStatsResult.cluster_name);
+      expect(stack.kibana).toBe(undefined); // not mocked for this test
+      expect(stack.data).toBe(undefined); // not mocked for this test
+
+      expect(cluster.version).toEqual(combinedStatsResult.version);
+      expect(cluster.cluster_stats).toEqual(combinedStatsResult.cluster_stats);
+      expect(Object.keys(cluster).indexOf('license')).toBeLessThan(0);
+      expect(Object.keys(stack).indexOf('xpack')).toBeLessThan(0);
+    });
+  });
+
+  describe('getLocalStats', () => {
+    it('returns expected object with kibana data', async () => {
+      const callCluster = jest.fn();
+      const usageCollection = mockUsageCollection(kibana);
+      const esClient = mockGetLocalStats(clusterInfo, clusterStats);
+      const response = await getLocalStats(
+        [{ clusterUuid: 'abc123' }],
+        { callCluster, usageCollection, esClient, start: '', end: '' },
+        context
+      );
+      const result = response[0];
+      expect(result.cluster_uuid).toEqual(combinedStatsResult.cluster_uuid);
+      expect(result.cluster_name).toEqual(combinedStatsResult.cluster_name);
+      expect(result.cluster_stats).toEqual(combinedStatsResult.cluster_stats);
+      expect(result.cluster_stats.nodes).toEqual(combinedStatsResult.cluster_stats.nodes);
+      expect(result.version).toBe('2.3.4');
+      expect(result.collection).toBe('local');
+      expect(Object.keys(result).indexOf('license')).toBeLessThan(0);
+      expect(Object.keys(result.stack_stats).indexOf('xpack')).toBeLessThan(0);
+    });
+
+    it('returns an empty array when no cluster uuid is provided', async () => {
+      const callCluster = jest.fn();
+      const usageCollection = mockUsageCollection(kibana);
+      const esClient = mockGetLocalStats(clusterInfo, clusterStats);
+      const response = await getLocalStats(
+        [],
+        { callCluster, usageCollection, esClient, start: '', end: '' },
+        context
+      );
+      expect(response).toBeDefined();
+      expect(response.length).toEqual(0);
+    });
+  });
+});
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts
index 98c83a3394628b..6244c6fac51d39 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts
@@ -40,8 +40,8 @@ export function handleLocalStats(
   // eslint-disable-next-line @typescript-eslint/naming-convention
   { cluster_name, cluster_uuid, version }: ESClusterInfo,
   { _nodes, cluster_name: clusterName, ...clusterStats }: any,
-  kibana: KibanaUsageStats,
-  dataTelemetry: DataTelemetryPayload,
+  kibana: KibanaUsageStats | undefined,
+  dataTelemetry: DataTelemetryPayload | undefined,
   context: StatsCollectionContext
 ) {
   return {
@@ -62,22 +62,25 @@ export type TelemetryLocalStats = ReturnType<typeof handleLocalStats>;
 
 /**
  * Get statistics for all products joined by Elasticsearch cluster.
+ * @param {Array} cluster uuids
+ * @param {Object} config contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end
+ * @param {Object} StatsCollectionContext contains logger and version (string)
  */
 export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
-  clustersDetails,
-  config,
-  context
+  clustersDetails, // array of cluster uuid's
+  config, // contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end
+  context // StatsCollectionContext contains logger and version (string)
 ) => {
-  const { callCluster, usageCollection } = config;
+  const { callCluster, usageCollection, esClient } = config;
 
   return await Promise.all(
     clustersDetails.map(async (clustersDetail) => {
       const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([
-        getClusterInfo(callCluster), // cluster info
-        getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
-        getNodesUsage(callCluster), // nodes_usage info
-        getKibana(usageCollection, callCluster),
-        getDataTelemetry(callCluster),
+        getClusterInfo(esClient), // cluster info
+        getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_)
+        getNodesUsage(esClient), // nodes_usage info
+        getKibana(usageCollection, callCluster, esClient),
+        getDataTelemetry(esClient),
       ]);
       return handleLocalStats(
         clusterInfo,
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts
index 4e4b0e11b79794..acf403ba254477 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts
@@ -19,6 +19,7 @@
 
 import { getNodesUsage } from './get_nodes_usage';
 import { TIMEOUT } from './constants';
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
 
 const mockedNodesFetchResponse = {
   cluster_name: 'test cluster',
@@ -44,37 +45,35 @@ const mockedNodesFetchResponse = {
     },
   },
 };
+
 describe('get_nodes_usage', () => {
-  it('calls fetchNodesUsage', async () => {
-    const callCluster = jest.fn();
-    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
-    await getNodesUsage(callCluster);
-    expect(callCluster).toHaveBeenCalledWith('transport.request', {
-      path: '/_nodes/usage',
-      method: 'GET',
-      query: {
-        timeout: TIMEOUT,
-      },
-    });
-  });
-  it('returns a modified array of node usage data', async () => {
-    const callCluster = jest.fn();
-    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
-    const result = await getNodesUsage(callCluster);
-    expect(result.nodes).toEqual([
-      {
-        aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
-        node_id: 'some_node_id',
-        rest_actions: {
-          create_index_action: 1,
-          document_get_action: 1,
-          nodes_info_action: 36,
-          nodes_usage_action: 1,
-          search_action: 19,
+  it('returns a modified array of nodes usage data', async () => {
+    const response = Promise.resolve({ body: mockedNodesFetchResponse });
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    esClient.nodes.usage.mockImplementationOnce(
+      // @ts-ignore
+      async (_params = { timeout: TIMEOUT }) => {
+        return response;
+      }
+    );
+    const item = await getNodesUsage(esClient);
+    expect(esClient.nodes.usage).toHaveBeenCalledWith({ timeout: TIMEOUT });
+    expect(item).toStrictEqual({
+      nodes: [
+        {
+          aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
+          node_id: 'some_node_id',
+          rest_actions: {
+            create_index_action: 1,
+            document_get_action: 1,
+            nodes_info_action: 36,
+            nodes_usage_action: 1,
+            search_action: 19,
+          },
+          since: 1588616945163,
+          timestamp: 1588617023177,
         },
-        since: 1588616945163,
-        timestamp: 1588617023177,
-      },
-    ]);
+      ],
+    });
   });
 });
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts
index c5c110fbb4149b..959840d0020a20 100644
--- a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { LegacyAPICaller } from 'kibana/server';
+import { ElasticsearchClient } from 'src/core/server';
 import { TIMEOUT } from './constants';
 
 export interface NodeAggregation {
@@ -44,7 +44,7 @@ export interface NodesFeatureUsageResponse {
 }
 
 export type NodesUsageGetter = (
-  callCluster: LegacyAPICaller
+  esClient: ElasticsearchClient
 ) => Promise<{ nodes: NodeObj[] | Array<{}> }>;
 /**
  * Get the nodes usage data from the connected cluster.
@@ -54,16 +54,12 @@ export type NodesUsageGetter = (
  * The Nodes usage API was introduced in v6.0.0
  */
 export async function fetchNodesUsage(
-  callCluster: LegacyAPICaller
+  esClient: ElasticsearchClient
 ): Promise<NodesFeatureUsageResponse> {
-  const response = await callCluster('transport.request', {
-    method: 'GET',
-    path: '/_nodes/usage',
-    query: {
-      timeout: TIMEOUT,
-    },
+  const { body } = await esClient.nodes.usage<NodesFeatureUsageResponse>({
+    timeout: TIMEOUT,
   });
-  return response;
+  return body;
 }
 
 /**
@@ -71,8 +67,8 @@ export async function fetchNodesUsage(
  * @param callCluster APICaller
  * @returns Object containing array of modified usage information with the node_id nested within the data for that node.
  */
-export const getNodesUsage: NodesUsageGetter = async (callCluster) => {
-  const result = await fetchNodesUsage(callCluster);
+export const getNodesUsage: NodesUsageGetter = async (esClient) => {
+  const result = await fetchNodesUsage(esClient);
   const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({
     ...(value as NodeObj),
     node_id: key,
diff --git a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts
index 438fcadad92555..9dac4900f5f108 100644
--- a/src/plugins/telemetry/server/telemetry_collection/register_collection.ts
+++ b/src/plugins/telemetry/server/telemetry_collection/register_collection.ts
@@ -38,16 +38,19 @@
 
 import { ILegacyClusterClient } from 'kibana/server';
 import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
+import { IClusterClient } from '../../../../../src/core/server';
 import { getLocalStats } from './get_local_stats';
 import { getClusterUuids } from './get_cluster_stats';
 import { getLocalLicense } from './get_local_license';
 
 export function registerCollection(
   telemetryCollectionManager: TelemetryCollectionManagerPluginSetup,
-  esCluster: ILegacyClusterClient
+  esCluster: ILegacyClusterClient,
+  esClientGetter: () => IClusterClient | undefined
 ) {
   telemetryCollectionManager.setCollection({
     esCluster,
+    esClientGetter,
     title: 'local',
     priority: 0,
     statsGetter: getLocalStats,
diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts
index 051bb3a11cb16c..e54e7451a670af 100644
--- a/src/plugins/telemetry_collection_manager/server/plugin.ts
+++ b/src/plugins/telemetry_collection_manager/server/plugin.ts
@@ -24,6 +24,7 @@ import {
   CoreStart,
   Plugin,
   Logger,
+  IClusterClient,
 } from '../../../core/server';
 
 import {
@@ -86,6 +87,7 @@ export class TelemetryCollectionManagerPlugin
       title,
       priority,
       esCluster,
+      esClientGetter,
       statsGetter,
       clusterDetailsGetter,
       licenseGetter,
@@ -105,6 +107,9 @@ export class TelemetryCollectionManagerPlugin
       if (!esCluster) {
         throw Error('esCluster name must be set for the getCluster method.');
       }
+      if (!esClientGetter) {
+        throw Error('esClientGetter method not set.');
+      }
       if (!clusterDetailsGetter) {
         throw Error('Cluster UUIds method is not set.');
       }
@@ -118,6 +123,7 @@ export class TelemetryCollectionManagerPlugin
         clusterDetailsGetter,
         esCluster,
         title,
+        esClientGetter,
       });
       this.usageGetterMethodPriority = priority;
     }
@@ -126,6 +132,7 @@ export class TelemetryCollectionManagerPlugin
   private getStatsCollectionConfig(
     config: StatsGetterConfig,
     collection: Collection,
+    collectionEsClient: IClusterClient,
     usageCollection: UsageCollectionSetup
   ): StatsCollectionConfig {
     const { start, end, request } = config;
@@ -133,8 +140,11 @@ export class TelemetryCollectionManagerPlugin
     const callCluster = config.unencrypted
       ? collection.esCluster.asScoped(request).callAsCurrentUser
       : collection.esCluster.callAsInternalUser;
-
-    return { callCluster, start, end, usageCollection };
+    // Scope the new elasticsearch Client appropriately and pass to the stats collection config
+    const esClient = config.unencrypted
+      ? collectionEsClient.asScoped(config.request).asCurrentUser
+      : collectionEsClient.asInternalUser;
+    return { callCluster, start, end, usageCollection, esClient };
   }
 
   private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) {
@@ -142,27 +152,33 @@ export class TelemetryCollectionManagerPlugin
       return [];
     }
     for (const collection of this.collections) {
-      const statsCollectionConfig = this.getStatsCollectionConfig(
-        config,
-        collection,
-        this.usageCollection
-      );
-      try {
-        const optInStats = await this.getOptInStatsForCollection(
+      // first fetch the client and make sure it's not undefined.
+      const collectionEsClient = collection.esClientGetter();
+      if (collectionEsClient !== undefined) {
+        const statsCollectionConfig = this.getStatsCollectionConfig(
+          config,
           collection,
-          optInStatus,
-          statsCollectionConfig
+          collectionEsClient,
+          this.usageCollection
         );
-        if (optInStats && optInStats.length) {
-          this.logger.debug(`Got Opt In stats using ${collection.title} collection.`);
-          if (config.unencrypted) {
-            return optInStats;
+
+        try {
+          const optInStats = await this.getOptInStatsForCollection(
+            collection,
+            optInStatus,
+            statsCollectionConfig
+          );
+          if (optInStats && optInStats.length) {
+            this.logger.debug(`Got Opt In stats using ${collection.title} collection.`);
+            if (config.unencrypted) {
+              return optInStats;
+            }
+            return encryptTelemetry(optInStats, { useProdKey: this.isDistributable });
           }
-          return encryptTelemetry(optInStats, { useProdKey: this.isDistributable });
+        } catch (err) {
+          this.logger.debug(`Failed to collect any opt in stats with registered collections.`);
+          // swallow error to try next collection;
         }
-      } catch (err) {
-        this.logger.debug(`Failed to collect any opt in stats with registered collections.`);
-        // swallow error to try next collection;
       }
     }
 
@@ -192,28 +208,32 @@ export class TelemetryCollectionManagerPlugin
       return [];
     }
     for (const collection of this.collections) {
-      const statsCollectionConfig = this.getStatsCollectionConfig(
-        config,
-        collection,
-        this.usageCollection
-      );
-      try {
-        const usageData = await this.getUsageForCollection(collection, statsCollectionConfig);
-        if (usageData.length) {
-          this.logger.debug(`Got Usage using ${collection.title} collection.`);
-          if (config.unencrypted) {
-            return usageData;
-          }
+      const collectionEsClient = collection.esClientGetter();
+      if (collectionEsClient !== undefined) {
+        const statsCollectionConfig = this.getStatsCollectionConfig(
+          config,
+          collection,
+          collectionEsClient,
+          this.usageCollection
+        );
+        try {
+          const usageData = await this.getUsageForCollection(collection, statsCollectionConfig);
+          if (usageData.length) {
+            this.logger.debug(`Got Usage using ${collection.title} collection.`);
+            if (config.unencrypted) {
+              return usageData;
+            }
 
-          return encryptTelemetry(usageData.filter(isClusterOptedIn), {
-            useProdKey: this.isDistributable,
-          });
+            return encryptTelemetry(usageData.filter(isClusterOptedIn), {
+              useProdKey: this.isDistributable,
+            });
+          }
+        } catch (err) {
+          this.logger.debug(
+            `Failed to collect any usage with registered collection ${collection.title}.`
+          );
+          // swallow error to try next collection;
         }
-      } catch (err) {
-        this.logger.debug(
-          `Failed to collect any usage with registered collection ${collection.title}.`
-        );
-        // swallow error to try next collection;
       }
     }
 
diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts
index 16f96c07fd8ead..44970df30fd16b 100644
--- a/src/plugins/telemetry_collection_manager/server/types.ts
+++ b/src/plugins/telemetry_collection_manager/server/types.ts
@@ -17,8 +17,15 @@
  * under the License.
  */
 
-import { LegacyAPICaller, Logger, KibanaRequest, ILegacyClusterClient } from 'kibana/server';
+import {
+  LegacyAPICaller,
+  Logger,
+  KibanaRequest,
+  ILegacyClusterClient,
+  IClusterClient,
+} from 'kibana/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { ElasticsearchClient } from '../../../../src/core/server';
 import { TelemetryCollectionManagerPlugin } from './plugin';
 
 export interface TelemetryCollectionManagerPluginSetup {
@@ -67,6 +74,7 @@ export interface StatsCollectionConfig {
   callCluster: LegacyAPICaller;
   start: string | number;
   end: string | number;
+  esClient: ElasticsearchClient;
 }
 
 export interface BasicStatsPayload {
@@ -100,7 +108,7 @@ export interface ESLicense {
 }
 
 export interface StatsCollectionContext {
-  logger: Logger;
+  logger: Logger | Console;
   version: string;
 }
 
@@ -130,6 +138,7 @@ export interface CollectionConfig<
   title: string;
   priority: number;
   esCluster: ILegacyClusterClient;
+  esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient but we assure that through a code check
   statsGetter: StatsGetter<CustomContext, T>;
   clusterDetailsGetter: ClusterDetailsGetter<CustomContext>;
   licenseGetter: LicenseGetter<CustomContext>;
@@ -145,5 +154,6 @@ export interface Collection<
   licenseGetter: LicenseGetter<CustomContext>;
   clusterDetailsGetter: ClusterDetailsGetter<CustomContext>;
   esCluster: ILegacyClusterClient;
+  esClientGetter: () => IClusterClient | undefined; // the collection could still return undefined for the es client getter.
   title: string;
 }
diff --git a/src/plugins/usage_collection/README.md b/src/plugins/usage_collection/README.md
index 0b1cca07de007d..d8edc5bb8d18a7 100644
--- a/src/plugins/usage_collection/README.md
+++ b/src/plugins/usage_collection/README.md
@@ -63,7 +63,7 @@ All you need to provide is a `type` for organizing your fields, `schema` field t
             total: 'long',
           },
         },
-        fetch: async (callCluster: APICluster) => {
+        fetch: async (callCluster: APICluster, esClient: IClusterClient) => {
 
         // query ES and get some data
         // summarize the data into a model
@@ -86,9 +86,9 @@ Some background:
 
 - `MY_USAGE_TYPE` can be any string. It usually matches the plugin name. As a safety mechanism, we double check there are no duplicates at the moment of registering the collector.
 - The `fetch` method needs to support multiple contexts in which it is called. For example, when stats are pulled from a Kibana Metricbeat module, the Beat calls Kibana's stats API to invoke usage collection.
-In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest`, where the request headers are expected to have read privilege on the entire `.kibana' index.
+In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest` or `esClient` wraps `asCurrentUser`, where the request headers are expected to have read privilege on the entire `.kibana' index. 
 
-Note: there will be many cases where you won't need to use the `callCluster` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below:
+Note: there will be many cases where you won't need to use the `callCluster` (or `esClient`) function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below:
 
 ```ts
 // server/plugin.ts
@@ -302,4 +302,4 @@ These saved objects are automatically consumed by the stats API and surfaced und
 By storing these metrics and their counts as key-value pairs, we can add more metrics without having
 to worry about exceeding the 1000-field soft limit in Elasticsearch.
 
-The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this. We are building a new way of reporting telemetry called [Pulse](../../../rfcs/text/0008_pulse.md) that will help on making these UI-Metrics easier to consume.
+The only caveat is that it makes it harder to consume in Kibana when analysing each entry in the array separately. In the telemetry team we are working to find a solution to this.
diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts
index d57700024c088f..365e1ce2013370 100644
--- a/src/plugins/usage_collection/server/collector/collector.ts
+++ b/src/plugins/usage_collection/server/collector/collector.ts
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { Logger, LegacyAPICaller } from 'kibana/server';
+import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server';
 
 export type CollectorFormatForBulkUpload<T, U> = (result: T) => { type: string; payload: U };
 
@@ -48,7 +48,7 @@ export interface CollectorOptions<T = unknown, U = T> {
   type: string;
   init?: Function;
   schema?: MakeSchemaFrom<Required<T>>; // Using Required to enforce all optional keys in the object
-  fetch: (callCluster: LegacyAPICaller) => Promise<T> | T;
+  fetch: (callCluster: LegacyAPICaller, esClient?: ElasticsearchClient) => Promise<T> | T;
   /*
    * A hook for allowing the fetched data payload to be organized into a typed
    * data model for internal bulk upload. See defaultFormatterForBulkUpload for
diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts
index 545642c5dcfa3b..3f943ad8bf2ffd 100644
--- a/src/plugins/usage_collection/server/collector/collector_set.test.ts
+++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts
@@ -21,7 +21,7 @@ import { noop } from 'lodash';
 import { Collector } from './collector';
 import { CollectorSet } from './collector_set';
 import { UsageCollector } from './usage_collector';
-import { loggingSystemMock } from '../../../../core/server/mocks';
+import { elasticsearchServiceMock, loggingSystemMock } from '../../../../core/server/mocks';
 
 const logger = loggingSystemMock.createLogger();
 
@@ -42,6 +42,7 @@ describe('CollectorSet', () => {
     });
 
     const mockCallCluster = jest.fn().mockResolvedValue({ passTest: 1000 });
+    const mockEsClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
 
     it('should throw an error if non-Collector type of object is registered', () => {
       const collectors = new CollectorSet({ logger });
@@ -85,7 +86,7 @@ describe('CollectorSet', () => {
         })
       );
 
-      const result = await collectors.bulkFetch(mockCallCluster);
+      const result = await collectors.bulkFetch(mockCallCluster, mockEsClient);
       expect(loggerSpies.debug).toHaveBeenCalledTimes(1);
       expect(loggerSpies.debug).toHaveBeenCalledWith(
         'Fetching data from MY_TEST_COLLECTOR collector'
@@ -110,7 +111,7 @@ describe('CollectorSet', () => {
 
       let result;
       try {
-        result = await collectors.bulkFetch(mockCallCluster);
+        result = await collectors.bulkFetch(mockCallCluster, mockEsClient);
       } catch (err) {
         // Do nothing
       }
@@ -128,7 +129,7 @@ describe('CollectorSet', () => {
         })
       );
 
-      const result = await collectors.bulkFetch(mockCallCluster);
+      const result = await collectors.bulkFetch(mockCallCluster, mockEsClient);
       expect(result).toStrictEqual([
         {
           type: 'MY_TEST_COLLECTOR',
@@ -146,7 +147,7 @@ describe('CollectorSet', () => {
         } as any)
       );
 
-      const result = await collectors.bulkFetch(mockCallCluster);
+      const result = await collectors.bulkFetch(mockCallCluster, mockEsClient);
       expect(result).toStrictEqual([
         {
           type: 'MY_TEST_COLLECTOR',
@@ -169,7 +170,7 @@ describe('CollectorSet', () => {
         })
       );
 
-      const result = await collectors.bulkFetch(mockCallCluster);
+      const result = await collectors.bulkFetch(mockCallCluster, mockEsClient);
       expect(result).toStrictEqual([
         {
           type: 'MY_TEST_COLLECTOR',
diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts
index fce17a46b71689..6861be7f4f76b1 100644
--- a/src/plugins/usage_collection/server/collector/collector_set.ts
+++ b/src/plugins/usage_collection/server/collector/collector_set.ts
@@ -18,7 +18,7 @@
  */
 
 import { snakeCase } from 'lodash';
-import { Logger, LegacyAPICaller } from 'kibana/server';
+import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server';
 import { Collector, CollectorOptions } from './collector';
 import { UsageCollector } from './usage_collector';
 
@@ -117,8 +117,12 @@ export class CollectorSet {
     return allReady;
   };
 
+  // all collections eventually pass through bulkFetch.
+  // the shape of the response is different when using the new ES client as is the error handling.
+  // We'll handle the refactor for using the new client in a follow up PR.
   public bulkFetch = async (
     callCluster: LegacyAPICaller,
+    esClient: ElasticsearchClient,
     collectors: Map<string, Collector<any, any>> = this.collectors
   ) => {
     const responses = await Promise.all(
@@ -127,7 +131,7 @@ export class CollectorSet {
         try {
           return {
             type: collector.type,
-            result: await collector.fetch(callCluster),
+            result: await collector.fetch(callCluster, esClient), // each collector must ensure they handle the response appropriately.
           };
         } catch (err) {
           this.logger.warn(err);
@@ -149,9 +153,9 @@ export class CollectorSet {
     return this.makeCollectorSetFromArray(filtered);
   };
 
-  public bulkFetchUsage = async (callCluster: LegacyAPICaller) => {
+  public bulkFetchUsage = async (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => {
     const usageCollectors = this.getFilteredCollectorSet((c) => c instanceof UsageCollector);
-    return await this.bulkFetch(callCluster, usageCollectors.collectors);
+    return await this.bulkFetch(callCluster, esClient, usageCollectors.collectors);
   };
 
   // convert an array of fetched stats results into key/object
diff --git a/src/plugins/usage_collection/server/routes/stats.ts b/src/plugins/usage_collection/server/routes/stats.ts
index 7c64c9f180319b..ef5da2eb11ba6a 100644
--- a/src/plugins/usage_collection/server/routes/stats.ts
+++ b/src/plugins/usage_collection/server/routes/stats.ts
@@ -24,6 +24,7 @@ import { Observable } from 'rxjs';
 import { first } from 'rxjs/operators';
 
 import {
+  ElasticsearchClient,
   IRouter,
   LegacyAPICaller,
   MetricsServiceSetup,
@@ -61,8 +62,11 @@ export function registerStatsRoute({
   metrics: MetricsServiceSetup;
   overallStatus$: Observable<ServiceStatus>;
 }) {
-  const getUsage = async (callCluster: LegacyAPICaller): Promise<any> => {
-    const usage = await collectorSet.bulkFetchUsage(callCluster);
+  const getUsage = async (
+    callCluster: LegacyAPICaller,
+    esClient: ElasticsearchClient
+  ): Promise<any> => {
+    const usage = await collectorSet.bulkFetchUsage(callCluster, esClient);
     return collectorSet.toObject(usage);
   };
 
@@ -96,13 +100,14 @@ export function registerStatsRoute({
       let extended;
       if (isExtended) {
         const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
+        const esClient = context.core.elasticsearch.client.asCurrentUser;
         const collectorsReady = await collectorSet.areAllCollectorsReady();
 
         if (shouldGetUsage && !collectorsReady) {
           return res.customError({ statusCode: 503, body: { message: STATS_NOT_READY_MESSAGE } });
         }
 
-        const usagePromise = shouldGetUsage ? getUsage(callCluster) : Promise.resolve({});
+        const usagePromise = shouldGetUsage ? getUsage(callCluster, esClient) : Promise.resolve({});
         const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]);
 
         let modifiedUsage = usage;
diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts
index 294f52cc3678ff..8d3248ddf43697 100644
--- a/x-pack/plugins/monitoring/server/plugin.ts
+++ b/x-pack/plugins/monitoring/server/plugin.ts
@@ -20,6 +20,7 @@ import {
   CoreStart,
   CustomHttpResponseOptions,
   ResponseError,
+  IClusterClient,
 } from 'kibana/server';
 import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
 import {
@@ -73,6 +74,7 @@ export class Plugin {
   private monitoringCore = {} as MonitoringCore;
   private legacyShimDependencies = {} as LegacyShimDependencies;
   private bulkUploader: IBulkUploader = {} as IBulkUploader;
+  private telemetryElasticsearchClient: IClusterClient | undefined;
 
   constructor(initializerContext: PluginInitializerContext) {
     this.initializerContext = initializerContext;
@@ -143,9 +145,14 @@ export class Plugin {
 
     // Initialize telemetry
     if (plugins.telemetryCollectionManager) {
-      registerMonitoringCollection(plugins.telemetryCollectionManager, this.cluster, {
-        maxBucketSize: config.ui.max_bucket_size,
-      });
+      registerMonitoringCollection(
+        plugins.telemetryCollectionManager,
+        this.cluster,
+        () => this.telemetryElasticsearchClient,
+        {
+          maxBucketSize: config.ui.max_bucket_size,
+        }
+      );
     }
 
     // Register collector objects for stats to show up in the APIs
@@ -229,7 +236,13 @@ export class Plugin {
     };
   }
 
-  start() {}
+  start({ elasticsearch }: CoreStart) {
+    // TODO: For the telemetry plugin to work, we need to provide the new ES client.
+    // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using
+    // the new client in Monitoring Telemetry collection yet, setting the local client allos progress for now.
+    // We will update the client in a follow up PR.
+    this.telemetryElasticsearchClient = elasticsearch.client;
+  }
 
   stop() {
     if (this.cluster) {
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts
index f0ad6399c6c72d..89f09d349014ff 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts
@@ -15,6 +15,7 @@ describe('get_all_stats', () => {
   const start = 0;
   const end = 1;
   const callCluster = sinon.stub();
+  const esClient = sinon.stub();
 
   const esClusters = [
     { cluster_uuid: 'a' },
@@ -176,6 +177,7 @@ describe('get_all_stats', () => {
           [{ clusterUuid: 'a' }],
           {
             callCluster: callCluster as any,
+            esClient: esClient as any,
             usageCollection: {} as any,
             start,
             end,
@@ -201,6 +203,7 @@ describe('get_all_stats', () => {
           [],
           {
             callCluster: callCluster as any,
+            esClient: esClient as any,
             usageCollection: {} as any,
             start,
             end,
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts
index 726db1706758d9..1170380b26ac8c 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts
@@ -21,7 +21,6 @@ type PromiseReturnType<T extends (...args: any[]) => any> = ReturnType<T> extend
 export interface CustomContext {
   maxBucketSize: number;
 }
-
 /**
  * Get statistics for all products joined by Elasticsearch cluster.
  * Returns the array of clusters joined with the Kibana and Logstash instances.
@@ -29,7 +28,7 @@ export interface CustomContext {
  */
 export const getAllStats: StatsGetter<CustomContext> = async (
   clustersDetails,
-  { callCluster, start, end },
+  { callCluster, start, end, esClient },
   { maxBucketSize }
 ) => {
   const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts
index 519dcc38875f50..b2f3cb6c61526c 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts
@@ -5,6 +5,7 @@
  */
 
 import sinon from 'sinon';
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
 import {
   getClusterUuids,
   fetchClusterUuids,
@@ -13,6 +14,7 @@ import {
 
 describe('get_cluster_uuids', () => {
   const callCluster = sinon.stub();
+  const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
   const response = {
     aggregations: {
       cluster_uuids: {
@@ -30,7 +32,7 @@ describe('get_cluster_uuids', () => {
     it('returns cluster UUIDs', async () => {
       callCluster.withArgs('search').returns(Promise.resolve(response));
       expect(
-        await getClusterUuids({ callCluster, start, end, usageCollection: {} as any }, {
+        await getClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, {
           maxBucketSize: 1,
         } as any)
       ).toStrictEqual(expectedUuids);
@@ -41,7 +43,7 @@ describe('get_cluster_uuids', () => {
     it('searches for clusters', async () => {
       callCluster.returns(Promise.resolve(response));
       expect(
-        await fetchClusterUuids({ callCluster, start, end, usageCollection: {} as any }, {
+        await fetchClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, {
           maxBucketSize: 1,
         } as any)
       ).toStrictEqual(response);
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts
index 1e5549fe4d8003..3648ae4bd8551b 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ILegacyCustomClusterClient } from 'kibana/server';
+import { ILegacyCustomClusterClient, IClusterClient } from 'kibana/server';
 import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
 import { getAllStats, CustomContext } from './get_all_stats';
 import { getClusterUuids } from './get_cluster_uuids';
@@ -13,10 +13,12 @@ import { getLicenses } from './get_licenses';
 export function registerMonitoringCollection(
   telemetryCollectionManager: TelemetryCollectionManagerPluginSetup,
   esCluster: ILegacyCustomClusterClient,
+  esClientGetter: () => IClusterClient | undefined,
   customContext: CustomContext
 ) {
   telemetryCollectionManager.setCollection({
     esCluster,
+    esClientGetter,
     title: 'monitoring',
     priority: 2,
     statsGetter: getAllStats,
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts
index 3f01d7423eded1..6ef44e325b0a70 100644
--- a/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts
+++ b/x-pack/plugins/telemetry_collection_xpack/server/plugin.ts
@@ -4,7 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from 'kibana/server';
+import {
+  PluginInitializerContext,
+  CoreSetup,
+  CoreStart,
+  Plugin,
+  IClusterClient,
+} from 'kibana/server';
 import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
 import { getClusterUuids, getLocalLicense } from '../../../../src/plugins/telemetry/server';
 import { getStatsWithXpack } from './telemetry_collection';
@@ -14,11 +20,13 @@ interface TelemetryCollectionXpackDepsSetup {
 }
 
 export class TelemetryCollectionXpackPlugin implements Plugin {
+  private elasticsearchClient?: IClusterClient;
   constructor(initializerContext: PluginInitializerContext) {}
 
   public setup(core: CoreSetup, { telemetryCollectionManager }: TelemetryCollectionXpackDepsSetup) {
     telemetryCollectionManager.setCollection({
       esCluster: core.elasticsearch.legacy.client,
+      esClientGetter: () => this.elasticsearchClient,
       title: 'local_xpack',
       priority: 1,
       statsGetter: getStatsWithXpack,
@@ -27,5 +35,7 @@ export class TelemetryCollectionXpackPlugin implements Plugin {
     });
   }
 
-  public start(core: CoreStart) {}
+  public start(core: CoreStart) {
+    this.elasticsearchClient = core.elasticsearch.client;
+  }
 }
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js
deleted file mode 100644
index eb03701fd195b5..00000000000000
--- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__tests__/get_xpack.js
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import expect from '@kbn/expect';
-import sinon from 'sinon';
-
-import { TIMEOUT } from '../constants';
-import { getXPackUsage } from '../get_xpack';
-
-function mockGetXPackUsage(callCluster, usage, req) {
-  callCluster
-    .withArgs(req, 'transport.request', {
-      method: 'GET',
-      path: '/_xpack/usage',
-      query: {
-        master_timeout: TIMEOUT,
-      },
-    })
-    .returns(usage);
-
-  callCluster
-    .withArgs('transport.request', {
-      method: 'GET',
-      path: '/_xpack/usage',
-      query: {
-        master_timeout: TIMEOUT,
-      },
-    })
-    .returns(usage);
-}
-
-describe('get_xpack', () => {
-  describe('getXPackUsage', () => {
-    it('uses callCluster to get /_xpack/usage API', () => {
-      const response = Promise.resolve({});
-      const callCluster = sinon.stub();
-
-      mockGetXPackUsage(callCluster, response);
-
-      expect(getXPackUsage(callCluster)).to.be(response);
-    });
-  });
-});
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts
index 24382fb89d3373..a4806cefeef3df 100644
--- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts
+++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { coreMock } from '../../../../../src/core/server/mocks';
+import { coreMock, elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
 import { getStatsWithXpack } from './get_stats_with_xpack';
 
 const kibana = {
@@ -55,32 +55,34 @@ const mockUsageCollection = (kibanaUsage = kibana) => ({
 
 describe('Telemetry Collection: Get Aggregated Stats', () => {
   test('OSS-like telemetry (no license nor X-Pack telemetry)', async () => {
-    const callCluster = jest.fn(async (method: string, options: { path?: string }) => {
-      switch (method) {
-        case 'transport.request':
-          if (options.path === '/_license' || options.path === '/_xpack/usage') {
-            // eslint-disable-next-line no-throw-literal
-            throw { statusCode: 404 };
-          } else if (options.path === '/_nodes/usage') {
-            return {
-              cluster_name: 'test cluster',
-              nodes: nodesUsage,
-            };
-          }
-          return {};
-        case 'info':
-          return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } };
-        default:
-          return {};
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    // mock for xpack.usage should throw a 404 for this test
+    esClient.xpack.usage.mockRejectedValue(new Error('Not Found'));
+    // mock for license should throw a 404 for this test
+    esClient.license.get.mockRejectedValue(new Error('Not Found'));
+    // mock for nodes usage should resolve for this test
+    esClient.nodes.usage.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      { body: { cluster_name: 'test cluster', nodes: nodesUsage } }
+    );
+    // mock for info should resolve for this test
+    esClient.info.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      {
+        body: {
+          cluster_uuid: 'test',
+          cluster_name: 'test',
+          version: { number: '8.0.0' },
+        },
       }
-    });
+    );
     const usageCollection = mockUsageCollection();
     const context = getContext();
 
     const stats = await getStatsWithXpack(
       [{ clusterUuid: '1234' }],
       {
-        callCluster,
+        esClient,
         usageCollection,
       } as any,
       context
@@ -93,36 +95,39 @@ describe('Telemetry Collection: Get Aggregated Stats', () => {
   });
 
   test('X-Pack telemetry (license + X-Pack)', async () => {
-    const callCluster = jest.fn(async (method: string, options: { path?: string }) => {
-      switch (method) {
-        case 'transport.request':
-          if (options.path === '/_license') {
-            return {
-              license: { type: 'basic' },
-            };
-          }
-          if (options.path === '/_xpack/usage') {
-            return {};
-          }
-          if (options.path === '/_nodes/usage') {
-            return {
-              cluster_name: 'test cluster',
-              nodes: nodesUsage,
-            };
-          }
-        case 'info':
-          return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } };
-        default:
-          return {};
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    // mock for license should return a basic license
+    esClient.license.get.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      { body: { license: { type: 'basic' } } }
+    );
+    // mock for xpack usage should return an empty object
+    esClient.xpack.usage.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      { body: {} }
+    );
+    // mock for nodes usage should return the cluster name and nodes usage
+    esClient.nodes.usage.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      { body: { cluster_name: 'test cluster', nodes: nodesUsage } }
+    );
+    esClient.info.mockResolvedValue(
+      // @ts-ignore we only care about the response body
+      {
+        body: {
+          cluster_uuid: 'test',
+          cluster_name: 'test',
+          version: { number: '8.0.0' },
+        },
       }
-    });
+    );
     const usageCollection = mockUsageCollection();
     const context = getContext();
 
     const stats = await getStatsWithXpack(
       [{ clusterUuid: '1234' }],
       {
-        callCluster,
+        esClient,
         usageCollection,
       } as any,
       context
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts
index 3fcd25c31e71e2..87e3d0a9613da1 100644
--- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts
+++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts
@@ -17,9 +17,9 @@ export const getStatsWithXpack: StatsGetter<{}, TelemetryAggregatedStats> = asyn
   config,
   context
 ) {
-  const { callCluster } = config;
+  const { esClient } = config;
   const clustersLocalStats = await getLocalStats(clustersDetails, config, context);
-  const xpack = await getXPackUsage(callCluster).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails.
+  const xpack = await getXPackUsage(esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails.
 
   return clustersLocalStats.map((localStats) => {
     if (xpack) {
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts
new file mode 100644
index 00000000000000..106df71e46c7e1
--- /dev/null
+++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.test.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+import { getXPackUsage } from './get_xpack';
+
+describe('get_xpack', () => {
+  describe('getXPackUsage', () => {
+    it('uses esClient to get /_xpack/usage API', async () => {
+      const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      // @ts-ignore we only care about the response body
+      esClient.xpack.usage.mockResolvedValue({ body: {} });
+      const result = await getXPackUsage(esClient);
+      expect(result).toEqual({});
+    });
+  });
+});
diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts
index 0ff75717c8b651..4dbf2052be28cd 100644
--- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts
+++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_xpack.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { LegacyAPICaller } from 'kibana/server';
+import { ElasticsearchClient } from 'src/core/server';
 import { TIMEOUT } from './constants';
 
 /**
@@ -14,12 +14,7 @@ import { TIMEOUT } from './constants';
  *
  * Like any X-Pack related API, X-Pack must installed for this to work.
  */
-export function getXPackUsage(callCluster: LegacyAPICaller) {
-  return callCluster('transport.request', {
-    method: 'GET',
-    path: '/_xpack/usage',
-    query: {
-      master_timeout: TIMEOUT,
-    },
-  });
+export async function getXPackUsage(esClient: ElasticsearchClient) {
+  const { body } = await esClient.xpack.usage({ master_timeout: TIMEOUT });
+  return body;
 }

From 341c1ace0dd85400821f74517fe5a435c9708c3a Mon Sep 17 00:00:00 2001
From: Brent Kimmel <bkimmel@users.noreply.github.com>
Date: Thu, 24 Sep 2020 22:34:48 -0400
Subject: [PATCH 07/22] [Security Solution][Resolver] 2 new functions to DAL
 (#78477)

* Added 2 new functions to DAL, adjusted old one to use new API

* update mocks to work with interface

* change to optional params

Co-authored-by: oatkiller <robert.austin@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../resolver/data_access_layer/factory.ts     | 48 ++++++++++++++++++-
 .../data_access_layer/mocks/emptify_mock.ts   |  1 +
 .../mocks/no_ancestors_two_children.ts        | 24 ++++++++++
 ..._children_in_index_called_awesome_index.ts | 27 +++++++++++
 ..._children_with_related_events_on_origin.ts | 47 +++++++++++++++---
 .../data_access_layer/mocks/pausify_mock.ts   |  1 +
 .../public/resolver/types.ts                  | 16 +++++++
 7 files changed, 155 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/factory.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/factory.ts
index 55d52d4ba32524..7b09e748c0c286 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/factory.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/factory.ts
@@ -11,6 +11,8 @@ import {
   ResolverRelatedEvents,
   ResolverTree,
   ResolverEntityIndex,
+  ResolverPaginatedEvents,
+  SafeResolverEvent,
 } from '../../../common/endpoint/types';
 
 /**
@@ -22,12 +24,54 @@ export function dataAccessLayerFactory(
   const dataAccessLayer: DataAccessLayer = {
     /**
      * Used to get non-process related events for a node.
+     * @deprecated use the new API (eventsWithEntityIDAndCategory & event) instead
      */
     async relatedEvents(entityID: string): Promise<ResolverRelatedEvents> {
-      return context.services.http.post(`/api/endpoint/resolver/${entityID}/events`, {
-        query: { events: 100 },
+      const response: ResolverPaginatedEvents = await context.services.http.post(
+        '/api/endpoint/resolver/events',
+        {
+          query: {},
+          body: JSON.stringify({
+            filter: `process.entity_id:"${entityID}" and not event.category:"process"`,
+          }),
+        }
+      );
+
+      return { ...response, entityID };
+    },
+
+    /**
+     * Return events that have `process.entity_id` that includes `entityID` and that have
+     * a `event.category` that includes `category`.
+     */
+    async eventsWithEntityIDAndCategory(
+      entityID: string,
+      category: string,
+      after?: string
+    ): Promise<ResolverPaginatedEvents> {
+      return context.services.http.post('/api/endpoint/resolver/events', {
+        query: { afterEvent: after },
+        body: JSON.stringify({
+          filter: `process.entity_id:"${entityID}" and event.category:"${category}"`,
+        }),
       });
     },
+
+    /**
+     * Return up to one event that has an `event.id` that includes `eventID`.
+     */
+    async event(eventID: string): Promise<SafeResolverEvent | null> {
+      const response: ResolverPaginatedEvents = await context.services.http.post(
+        '/api/endpoint/resolver/events',
+        {
+          query: {},
+          body: JSON.stringify({ filter: `event.id:"${eventID}"` }),
+        }
+      );
+      const [oneEvent] = response.events;
+      return oneEvent ?? null;
+    },
+
     /**
      * Used to get descendant and ancestor process events for a node.
      */
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/emptify_mock.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/emptify_mock.ts
index 631eab18fc014e..88a3052a61f743 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/emptify_mock.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/emptify_mock.ts
@@ -44,6 +44,7 @@ export function emptifyMock<T>(
   return {
     metadata,
     dataAccessLayer: {
+      ...dataAccessLayer,
       /**
        * Fetch related events for an entity ID
        */
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
index fd086bd9b984e6..09625e5726b1d5 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
@@ -6,6 +6,7 @@
 
 import {
   ResolverRelatedEvents,
+  SafeResolverEvent,
   ResolverTree,
   ResolverEntityIndex,
 } from '../../../../common/endpoint/types';
@@ -58,6 +59,29 @@ export function noAncestorsTwoChildren(): { dataAccessLayer: DataAccessLayer; me
         });
       },
 
+      /**
+       * Return events that have `process.entity_id` that includes `entityID` and that have
+       * a `event.category` that includes `category`.
+       */
+      async eventsWithEntityIDAndCategory(
+        entityID: string,
+        category: string,
+        after?: string
+      ): Promise<{
+        events: SafeResolverEvent[];
+        nextEvent: string | null;
+      }> {
+        const events: SafeResolverEvent[] = [];
+        return {
+          events,
+          nextEvent: null,
+        };
+      },
+
+      async event(_eventID: string): Promise<SafeResolverEvent | null> {
+        return null;
+      },
+
       /**
        * Fetch a ResolverTree for a entityID
        */
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
index 86450b25eb1dad..3bbe4bcf510607 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
@@ -8,6 +8,7 @@ import {
   ResolverRelatedEvents,
   ResolverTree,
   ResolverEntityIndex,
+  SafeResolverEvent,
 } from '../../../../common/endpoint/types';
 import { mockEndpointEvent } from '../../mocks/endpoint_event';
 import { mockTreeWithNoAncestorsAnd2Children } from '../../mocks/resolver_tree';
@@ -69,6 +70,32 @@ export function noAncestorsTwoChildenInIndexCalledAwesomeIndex(): {
         });
       },
 
+      async eventsWithEntityIDAndCategory(
+        entityID: string,
+        category,
+        after?: string
+      ): Promise<{
+        events: SafeResolverEvent[];
+        nextEvent: string | null;
+      }> {
+        return {
+          events: [
+            mockEndpointEvent({
+              entityID,
+              eventCategory: category,
+            }),
+          ],
+          nextEvent: null,
+        };
+      },
+
+      async event(eventID: string): Promise<SafeResolverEvent | null> {
+        return mockEndpointEvent({
+          entityID: metadata.entityIDs.origin,
+          eventID,
+        });
+      },
+
       /**
        * Fetch a ResolverTree for a entityID
        */
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
index ec773a09ae8e04..6fb84eaf7fda63 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
@@ -10,7 +10,9 @@ import {
   ResolverRelatedEvents,
   ResolverTree,
   ResolverEntityIndex,
+  SafeResolverEvent,
 } from '../../../../common/endpoint/types';
+import * as eventModel from '../../../../common/endpoint/models/event';
 
 interface Metadata {
   /**
@@ -56,31 +58,62 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOrigin(): {
       /**
        * Fetch related events for an entity ID
        */
-      relatedEvents(entityID: string): Promise<ResolverRelatedEvents> {
+      async relatedEvents(entityID: string): Promise<ResolverRelatedEvents> {
         /**
          * Respond with the mocked related events when the origin's related events are fetched.
          **/
         const events = entityID === metadata.entityIDs.origin ? tree.relatedEvents.events : [];
 
-        return Promise.resolve({
+        return {
           entityID,
           events,
           nextEvent: null,
-        });
+        };
+      },
+
+      /**
+       * Any of the origin's related events by category.
+       * `entityID` must match the origin node's `process.entity_id`.
+       * Does not respect the `_after` parameter.
+       */
+      async eventsWithEntityIDAndCategory(
+        entityID: string,
+        category: string,
+        after?: string
+      ): Promise<{ events: SafeResolverEvent[]; nextEvent: string | null }> {
+        const events =
+          entityID === metadata.entityIDs.origin
+            ? tree.relatedEvents.events.filter((event) =>
+                eventModel.eventCategory(event).includes(category)
+              )
+            : [];
+        return {
+          events,
+          nextEvent: null,
+        };
+      },
+
+      /**
+       * Any of the origin's related events by event.id
+       */
+      async event(eventID: string): Promise<SafeResolverEvent | null> {
+        return (
+          tree.relatedEvents.events.find((event) => eventModel.eventID(event) === eventID) ?? null
+        );
       },
 
       /**
        * Fetch a ResolverTree for a entityID
        */
-      resolverTree(): Promise<ResolverTree> {
-        return Promise.resolve(tree);
+      async resolverTree(): Promise<ResolverTree> {
+        return tree;
       },
 
       /**
        * Get entities matching a document.
        */
-      entities(): Promise<ResolverEntityIndex> {
-        return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]);
+      async entities(): Promise<ResolverEntityIndex> {
+        return [{ entity_id: metadata.entityIDs.origin }];
       },
     },
   };
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/pausify_mock.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/pausify_mock.ts
index 6a4955b104b8fb..a3ec667385470a 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/pausify_mock.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/pausify_mock.ts
@@ -89,6 +89,7 @@ export function pausifyMock<T>({
       }
     },
     dataAccessLayer: {
+      ...dataAccessLayer,
       /**
        * Fetch related events for an entity ID
        */
diff --git a/x-pack/plugins/security_solution/public/resolver/types.ts b/x-pack/plugins/security_solution/public/resolver/types.ts
index 4dc614abe3345d..64147dd8feb75c 100644
--- a/x-pack/plugins/security_solution/public/resolver/types.ts
+++ b/x-pack/plugins/security_solution/public/resolver/types.ts
@@ -16,6 +16,7 @@ import {
   ResolverTree,
   ResolverEntityIndex,
   SafeResolverEvent,
+  ResolverPaginatedEvents,
 } from '../../common/endpoint/types';
 
 /**
@@ -503,6 +504,21 @@ export interface DataAccessLayer {
    */
   relatedEvents: (entityID: string) => Promise<ResolverRelatedEvents>;
 
+  /**
+   * Return events that have `process.entity_id` that includes `entityID` and that have
+   * a `event.category` that includes `category`.
+   */
+  eventsWithEntityIDAndCategory: (
+    entityID: string,
+    category: string,
+    after?: string
+  ) => Promise<ResolverPaginatedEvents>;
+
+  /**
+   * Return up to one event that has an `event.id` that includes `eventID`.
+   */
+  event: (eventID: string) => Promise<SafeResolverEvent | null>;
+
   /**
    * Fetch a ResolverTree for a entityID
    */

From 012fa42ee1a8261150326c363170bd8966a2ccec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= <patryk.kopycinski@elastic.co>
Date: Fri, 25 Sep 2020 14:15:41 +0200
Subject: [PATCH 08/22] [Security Solution] Fix app layout (#76668)

---
 test/functional/services/common/browser.ts    |   4 +
 .../security_solution/common/constants.ts     |   1 +
 .../cypress/integration/events_viewer.spec.ts |   3 +-
 .../cypress/screens/hosts/events.ts           |   2 -
 .../public/app/home/index.tsx                 |  24 +--
 .../cases/components/case_view/index.tsx      |   2 +-
 .../cases/components/wrappers/index.tsx       |   3 +-
 .../common/components/events_viewer/index.tsx |   1 +
 .../components/exit_full_screen/index.tsx     |   9 +-
 .../common/components/header_global/index.tsx | 156 +++++++++---------
 .../public/common/components/page/index.tsx   |  54 ++----
 .../common/components/wrapper_page/index.tsx  |   6 +
 .../page_objects/policy_page.ts               |   2 +
 13 files changed, 133 insertions(+), 134 deletions(-)

diff --git a/test/functional/services/common/browser.ts b/test/functional/services/common/browser.ts
index daf1659f0cfe13..f5fb54c72177fb 100644
--- a/test/functional/services/common/browser.ts
+++ b/test/functional/services/common/browser.ts
@@ -474,6 +474,10 @@ export async function BrowserProvider({ getService }: FtrProviderContext) {
       return parseInt(scrollSize, 10);
     }
 
+    public async scrollTop() {
+      await driver.executeScript('document.documentElement.scrollTop = 0');
+    }
+
     // return promise with REAL scroll position
     public async setScrollTop(scrollSize: number | string) {
       await driver.executeScript('document.body.scrollTop = ' + scrollSize);
diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
index a93d2817fbbb3e..2910f02a187f48 100644
--- a/x-pack/plugins/security_solution/common/constants.ts
+++ b/x-pack/plugins/security_solution/common/constants.ts
@@ -34,6 +34,7 @@ export const DEFAULT_INTERVAL_TYPE = 'manual';
 export const DEFAULT_INTERVAL_VALUE = 300000; // ms
 export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges';
 export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled';
+export const GLOBAL_HEADER_HEIGHT = 98; // px
 export const FILTERS_GLOBAL_HEIGHT = 109; // px
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled';
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51';
diff --git a/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts b/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts
index d193330dc54ff4..4e2edcb282cfc4 100644
--- a/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/events_viewer.spec.ts
@@ -10,7 +10,6 @@ import {
   FIELDS_BROWSER_SELECTED_CATEGORY_TITLE,
 } from '../screens/fields_browser';
 import {
-  EVENTS_PAGE,
   HEADER_SUBTITLE,
   HOST_GEO_CITY_NAME_HEADER,
   HOST_GEO_COUNTRY_NAME_HEADER,
@@ -173,7 +172,7 @@ describe.skip('Events Viewer', () => {
       const expectedOrderAfterDragAndDrop =
         'message@timestamphost.nameevent.moduleevent.datasetevent.actionuser.namesource.ipdestination.ip';
 
-      cy.get(EVENTS_PAGE).scrollTo('bottom');
+      cy.scrollTo('bottom');
       cy.get(HEADERS_GROUP).invoke('text').should('equal', originalColumnOrder);
       dragAndDropColumn({ column: 0, newPosition: 1 });
       cy.get(HEADERS_GROUP).invoke('text').should('equal', expectedOrderAfterDragAndDrop);
diff --git a/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts b/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts
index 05f517b5de6624..4b1ca19bd96fef 100644
--- a/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts
+++ b/x-pack/plugins/security_solution/cypress/screens/hosts/events.ts
@@ -6,8 +6,6 @@
 
 export const CLOSE_MODAL = '[data-test-subj="modal-inspect-close"]';
 
-export const EVENTS_PAGE = '[data-test-subj="pageContainer"]';
-
 export const EVENTS_VIEWER_FIELDS_BUTTON =
   '[data-test-subj="events-viewer-panel"] [data-test-subj="show-field-browser"]';
 
diff --git a/x-pack/plugins/security_solution/public/app/home/index.tsx b/x-pack/plugins/security_solution/public/app/home/index.tsx
index e0dea199e78ff4..24e25470feb3b3 100644
--- a/x-pack/plugins/security_solution/public/app/home/index.tsx
+++ b/x-pack/plugins/security_solution/public/app/home/index.tsx
@@ -10,6 +10,7 @@ import styled from 'styled-components';
 import { TimelineId } from '../../../common/types/timeline';
 import { DragDropContextWrapper } from '../../common/components/drag_and_drop/drag_drop_context_wrapper';
 import { Flyout } from '../../timelines/components/flyout';
+import { SecuritySolutionAppWrapper } from '../../common/components/page';
 import { HeaderGlobal } from '../../common/components/header_global';
 import { HelpMenu } from '../../common/components/help_menu';
 import { AutoSaveWarningMsg } from '../../timelines/components/timeline/auto_save_warning';
@@ -20,18 +21,17 @@ import { useInitSourcerer, useSourcererScope } from '../../common/containers/sou
 import { useKibana } from '../../common/lib/kibana';
 import { DETECTIONS_SUB_PLUGIN_ID } from '../../../common/constants';
 import { SourcererScopeName } from '../../common/store/sourcerer/model';
+import { useThrottledResizeObserver } from '../../common/components/utils';
 
-const SecuritySolutionAppWrapper = styled.div`
+const Main = styled.main.attrs<{ paddingTop: number }>(({ paddingTop }) => ({
+  style: {
+    paddingTop: `${paddingTop}px`,
+  },
+}))<{ paddingTop: number }>`
+  overflow: auto;
   display: flex;
   flex-direction: column;
-  height: 100%;
-  width: 100%;
-`;
-SecuritySolutionAppWrapper.displayName = 'SecuritySolutionAppWrapper';
-
-const Main = styled.main`
-  overflow: auto;
-  flex: 1;
+  flex: 1 1 auto;
 `;
 
 Main.displayName = 'Main';
@@ -45,7 +45,7 @@ interface HomePageProps {
 const HomePageComponent: React.FC<HomePageProps> = ({ children }) => {
   const { application } = useKibana().services;
   const subPluginId = useRef<string>('');
-
+  const { ref, height = 0 } = useThrottledResizeObserver(300);
   application.currentAppId$.subscribe((appId) => {
     subPluginId.current = appId ?? '';
   });
@@ -61,9 +61,9 @@ const HomePageComponent: React.FC<HomePageProps> = ({ children }) => {
 
   return (
     <SecuritySolutionAppWrapper>
-      <HeaderGlobal />
+      <HeaderGlobal ref={ref} />
 
-      <Main data-test-subj="pageContainer">
+      <Main paddingTop={height} data-test-subj="pageContainer">
         <DragDropContextWrapper browserFields={browserFields}>
           <UseUrlState indexPattern={indexPattern} navTabs={navTabs} />
           {indicesExist && showTimeline && (
diff --git a/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx b/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx
index ad113d3e7e7372..750ff49cd700cd 100644
--- a/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx
+++ b/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx
@@ -402,7 +402,7 @@ export const CaseView = React.memo(({ caseId, userCanCrud }: Props) => {
   }
   if (isLoading) {
     return (
-      <MyEuiFlexGroup justifyContent="center" alignItems="center">
+      <MyEuiFlexGroup gutterSize="none" justifyContent="center" alignItems="center">
         <EuiFlexItem grow={false}>
           <EuiLoadingSpinner data-test-subj="case-view-loading" size="xl" />
         </EuiFlexItem>
diff --git a/x-pack/plugins/security_solution/public/cases/components/wrappers/index.tsx b/x-pack/plugins/security_solution/public/cases/components/wrappers/index.tsx
index 06715514e01bf6..b89a7e8eefec32 100644
--- a/x-pack/plugins/security_solution/public/cases/components/wrappers/index.tsx
+++ b/x-pack/plugins/security_solution/public/cases/components/wrappers/index.tsx
@@ -10,8 +10,7 @@ import { gutterTimeline } from '../../../common/lib/helpers';
 export const WhitePageWrapper = styled.div`
   background-color: ${({ theme }) => theme.eui.euiColorEmptyShade};
   border-top: ${({ theme }) => theme.eui.euiBorderThin};
-  height: 100%;
-  min-height: 100vh;
+  flex: 1 1 auto;
 `;
 
 export const SectionWrapper = styled.div`
diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx
index cd43c7e4930657..c53d311dc1361c 100644
--- a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx
@@ -29,6 +29,7 @@ const DEFAULT_EVENTS_VIEWER_HEIGHT = 652;
 
 const FullScreenContainer = styled.div<{ $isFullScreen: boolean }>`
   height: ${({ $isFullScreen }) => ($isFullScreen ? '100%' : `${DEFAULT_EVENTS_VIEWER_HEIGHT}px`)};
+  flex: 1 1 auto;
   display: flex;
   width: 100%;
 `;
diff --git a/x-pack/plugins/security_solution/public/common/components/exit_full_screen/index.tsx b/x-pack/plugins/security_solution/public/common/components/exit_full_screen/index.tsx
index 8c5ad95a8de0e9..cd4740bc8c4649 100644
--- a/x-pack/plugins/security_solution/public/common/components/exit_full_screen/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/exit_full_screen/index.tsx
@@ -6,11 +6,16 @@
 
 import { EuiButton, EuiWindowEvent } from '@elastic/eui';
 import React, { useCallback } from 'react';
+import styled from 'styled-components';
 
 import { useFullScreen } from '../../../common/containers/use_full_screen';
 
 import * as i18n from './translations';
 
+const StyledEuiButton = styled(EuiButton)`
+  margin: ${({ theme }) => theme.eui.paddingSizes.s};
+`;
+
 export const ExitFullScreen: React.FC = () => {
   const { globalFullScreen, setGlobalFullScreen } = useFullScreen();
 
@@ -36,14 +41,14 @@ export const ExitFullScreen: React.FC = () => {
   return (
     <>
       <EuiWindowEvent event="keydown" handler={onKeyDown} />
-      <EuiButton
+      <StyledEuiButton
         data-test-subj="exit-full-screen"
         iconType="fullScreen"
         isDisabled={!globalFullScreen}
         onClick={exitFullScreen}
       >
         {i18n.EXIT_FULL_SCREEN}
-      </EuiButton>
+      </StyledEuiButton>
     </>
   );
 };
diff --git a/x-pack/plugins/security_solution/public/common/components/header_global/index.tsx b/x-pack/plugins/security_solution/public/common/components/header_global/index.tsx
index 5b4dd2e9728bba..0c6a54d4434d21 100644
--- a/x-pack/plugins/security_solution/public/common/components/header_global/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/header_global/index.tsx
@@ -6,7 +6,7 @@
 
 import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui';
 import { pickBy } from 'lodash/fp';
-import React, { useCallback } from 'react';
+import React, { forwardRef, useCallback } from 'react';
 import styled from 'styled-components';
 import { OutPortal } from 'react-reverse-portal';
 
@@ -24,30 +24,37 @@ import { APP_ID, ADD_DATA_PATH, APP_DETECTIONS_PATH } from '../../../../common/c
 import { useGlobalHeaderPortal } from '../../hooks/use_global_header_portal';
 import { LinkAnchor } from '../links';
 
-const Wrapper = styled.header<{ $globalFullScreen: boolean }>`
-  ${({ $globalFullScreen, theme }) => `
+const Wrapper = styled.header`
+  ${({ theme }) => `
     background: ${theme.eui.euiColorEmptyShade};
     border-bottom: ${theme.eui.euiBorderThin};
-    padding-top: ${$globalFullScreen ? theme.eui.paddingSizes.s : theme.eui.paddingSizes.m};
     width: 100%;
     z-index: ${theme.eui.euiZNavigation};
+    position: fixed;
   `}
 `;
 Wrapper.displayName = 'Wrapper';
 
+const WrapperContent = styled.div<{ $globalFullScreen: boolean }>`
+  display: ${({ $globalFullScreen }) => ($globalFullScreen ? 'none' : 'block')};
+  padding-top: ${({ $globalFullScreen, theme }) =>
+    $globalFullScreen ? theme.eui.paddingSizes.s : theme.eui.paddingSizes.m};
+`;
+
+WrapperContent.displayName = 'WrapperContent';
+
 const FlexItem = styled(EuiFlexItem)`
   min-width: 0;
 `;
 FlexItem.displayName = 'FlexItem';
 
-const FlexGroup = styled(EuiFlexGroup)<{ $globalFullScreen: boolean; $hasSibling: boolean }>`
-  ${({ $globalFullScreen, $hasSibling, theme }) => `
+const FlexGroup = styled(EuiFlexGroup)<{ $hasSibling: boolean }>`
+  ${({ $hasSibling, theme }) => `
     border-bottom: ${theme.eui.euiBorderThin};
     margin-bottom: 1px;
     padding-bottom: 4px;
     padding-left: ${theme.eui.paddingSizes.l};
     padding-right: ${gutterTimeline};
-    ${$globalFullScreen ? 'display: none;' : ''}
     ${$hasSibling ? `border-bottom: ${theme.eui.euiBorderThin};` : 'border-bottom-width: 0px;'}
   `}
 `;
@@ -56,77 +63,74 @@ FlexGroup.displayName = 'FlexGroup';
 interface HeaderGlobalProps {
   hideDetectionEngine?: boolean;
 }
-export const HeaderGlobal = React.memo<HeaderGlobalProps>(({ hideDetectionEngine = false }) => {
-  const { globalHeaderPortalNode } = useGlobalHeaderPortal();
-  const { globalFullScreen } = useFullScreen();
-  const search = useGetUrlSearch(navTabs.overview);
-  const { application, http } = useKibana().services;
-  const { navigateToApp } = application;
-  const basePath = http.basePath.get();
-  const goToOverview = useCallback(
-    (ev) => {
-      ev.preventDefault();
-      navigateToApp(`${APP_ID}:${SecurityPageName.overview}`, { path: search });
-    },
-    [navigateToApp, search]
-  );
-
-  return (
-    <Wrapper className="siemHeaderGlobal" $globalFullScreen={globalFullScreen}>
-      <FlexGroup
-        alignItems="center"
-        $globalFullScreen={globalFullScreen}
-        $hasSibling={globalHeaderPortalNode.hasChildNodes()}
-        justifyContent="spaceBetween"
-        wrap
-      >
-        <>
-          <FlexItem>
-            <EuiFlexGroup alignItems="center" responsive={false}>
-              <FlexItem grow={false}>
-                <LinkAnchor onClick={goToOverview} href={getAppOverviewUrl(search)}>
-                  <EuiIcon aria-label={i18n.SECURITY_SOLUTION} type="logoSecurity" size="l" />
-                </LinkAnchor>
-              </FlexItem>
+export const HeaderGlobal = React.memo(
+  forwardRef<HTMLDivElement, HeaderGlobalProps>(({ hideDetectionEngine = false }, ref) => {
+    const { globalHeaderPortalNode } = useGlobalHeaderPortal();
+    const { globalFullScreen } = useFullScreen();
+    const search = useGetUrlSearch(navTabs.overview);
+    const { application, http } = useKibana().services;
+    const { navigateToApp } = application;
+    const basePath = http.basePath.get();
+    const goToOverview = useCallback(
+      (ev) => {
+        ev.preventDefault();
+        navigateToApp(`${APP_ID}:${SecurityPageName.overview}`, { path: search });
+      },
+      [navigateToApp, search]
+    );
+    return (
+      <Wrapper ref={ref} className="siemHeaderGlobal">
+        <WrapperContent $globalFullScreen={globalFullScreen}>
+          <FlexGroup
+            alignItems="center"
+            $hasSibling={globalHeaderPortalNode.hasChildNodes()}
+            justifyContent="spaceBetween"
+            wrap
+          >
+            <FlexItem>
+              <EuiFlexGroup alignItems="center" responsive={false}>
+                <FlexItem grow={false}>
+                  <LinkAnchor onClick={goToOverview} href={getAppOverviewUrl(search)}>
+                    <EuiIcon aria-label={i18n.SECURITY_SOLUTION} type="logoSecurity" size="l" />
+                  </LinkAnchor>
+                </FlexItem>
 
-              <FlexItem component="nav">
-                <SiemNavigation
-                  display="condensed"
-                  navTabs={
-                    hideDetectionEngine
-                      ? pickBy((_, key) => key !== SecurityPageName.detections, navTabs)
-                      : navTabs
-                  }
-                />
-              </FlexItem>
-            </EuiFlexGroup>
-          </FlexItem>
+                <FlexItem component="nav">
+                  <SiemNavigation
+                    display="condensed"
+                    navTabs={
+                      hideDetectionEngine
+                        ? pickBy((_, key) => key !== SecurityPageName.detections, navTabs)
+                        : navTabs
+                    }
+                  />
+                </FlexItem>
+              </EuiFlexGroup>
+            </FlexItem>
+            <FlexItem grow={false}>
+              <EuiFlexGroup alignItems="center" gutterSize="s" responsive={false} wrap>
+                {window.location.pathname.includes(APP_DETECTIONS_PATH) && (
+                  <FlexItem grow={false}>
+                    <MlPopover />
+                  </FlexItem>
+                )}
 
-          <FlexItem grow={false}>
-            <EuiFlexGroup alignItems="center" gutterSize="s" responsive={false} wrap>
-              {window.location.pathname.includes(APP_DETECTIONS_PATH) && (
                 <FlexItem grow={false}>
-                  <MlPopover />
+                  <EuiButtonEmpty
+                    data-test-subj="add-data"
+                    href={`${basePath}${ADD_DATA_PATH}`}
+                    iconType="plusInCircle"
+                  >
+                    {i18n.BUTTON_ADD_DATA}
+                  </EuiButtonEmpty>
                 </FlexItem>
-              )}
-
-              <FlexItem grow={false}>
-                <EuiButtonEmpty
-                  data-test-subj="add-data"
-                  href={`${basePath}${ADD_DATA_PATH}`}
-                  iconType="plusInCircle"
-                >
-                  {i18n.BUTTON_ADD_DATA}
-                </EuiButtonEmpty>
-              </FlexItem>
-            </EuiFlexGroup>
-          </FlexItem>
-        </>
-      </FlexGroup>
-      <div>
-        <OutPortal node={globalHeaderPortalNode} />
-      </div>
-    </Wrapper>
-  );
-});
+              </EuiFlexGroup>
+            </FlexItem>
+          </FlexGroup>
+          <OutPortal node={globalHeaderPortalNode} />
+        </WrapperContent>
+      </Wrapper>
+    );
+  })
+);
 HeaderGlobal.displayName = 'HeaderGlobal';
diff --git a/x-pack/plugins/security_solution/public/common/components/page/index.tsx b/x-pack/plugins/security_solution/public/common/components/page/index.tsx
index 140429dc4abd78..8a8eda3e20185f 100644
--- a/x-pack/plugins/security_solution/public/common/components/page/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/page/index.tsx
@@ -8,27 +8,36 @@ import { EuiBadge, EuiDescriptionList, EuiFlexGroup, EuiIcon, EuiPage } from '@e
 import styled, { createGlobalStyle } from 'styled-components';
 
 import {
+  GLOBAL_HEADER_HEIGHT,
   FULL_SCREEN_TOGGLED_CLASS_NAME,
   SCROLLING_DISABLED_CLASS_NAME,
 } from '../../../../common/constants';
 
+export const SecuritySolutionAppWrapper = styled.div`
+  display: flex;
+  flex-direction: column;
+  flex: 1 1 auto;
+  width: 100%;
+`;
+SecuritySolutionAppWrapper.displayName = 'SecuritySolutionAppWrapper';
+
 /*
   SIDE EFFECT: the following `createGlobalStyle` overrides default styling in angular code that was not theme-friendly
   and `EuiPopover`, `EuiToolTip` global styles
 */
 export const AppGlobalStyle = createGlobalStyle<{ theme: { eui: { euiColorPrimary: string } } }>`
-  /* dirty hack to fix draggables with tooltip on FF */
-  body#siem-app {
-    position: static;
-  }
-  /* end of dirty hack to fix draggables with tooltip on FF */
-
   div.app-wrapper {
     background-color: rgba(0,0,0,0);
   }
 
   div.application {
     background-color: rgba(0,0,0,0);
+
+    // Security App wrapper
+    > div {
+      display: flex;
+      flex: 1 1 auto;
+    }
   }
 
   .euiPopover__panel.euiPopover__panel-isOpen {
@@ -67,37 +76,8 @@ export const AppGlobalStyle = createGlobalStyle<{ theme: { eui: { euiColorPrimar
     ${({ theme }) => `background-color: ${theme.eui.euiColorPrimary} !important`};
   }
 
-  body {
-    overflow-y: hidden;
-  }
-
-  #kibana-body {
-    height: 100%;
-    overflow-y: hidden;
-
-    > .content {
-      height: 100%;
-
-      > .app-wrapper {
-        height: 100%;
-
-        > .app-wrapper-panel {
-          height: 100%;
-
-          > .application {
-            height: 100%;
-
-            > div {
-               height: 100%;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  .${SCROLLING_DISABLED_CLASS_NAME} #kibana-body {
-    overflow-y: hidden;
+  .${SCROLLING_DISABLED_CLASS_NAME} ${SecuritySolutionAppWrapper} {
+    max-height: calc(100vh - ${GLOBAL_HEADER_HEIGHT}px);
   }
 `;
 
diff --git a/x-pack/plugins/security_solution/public/common/components/wrapper_page/index.tsx b/x-pack/plugins/security_solution/public/common/components/wrapper_page/index.tsx
index f3136b0a40b3e4..0908c887d25f6e 100644
--- a/x-pack/plugins/security_solution/public/common/components/wrapper_page/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/wrapper_page/index.tsx
@@ -28,6 +28,9 @@ const Wrapper = styled.div`
 
   &.siemWrapperPage--fullHeight {
     height: 100%;
+    display: flex;
+    flex-direction: column;
+    flex: 1 1 auto;
   }
 
   &.siemWrapperPage--withTimeline {
@@ -36,6 +39,9 @@ const Wrapper = styled.div`
 
   &.siemWrapperPage--noPadding {
     padding: 0;
+    display: flex;
+    flex-direction: column;
+    flex: 1 1 auto;
   }
 `;
 
diff --git a/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts b/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts
index d661b3097bd354..92571e5c275665 100644
--- a/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts
+++ b/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts
@@ -9,6 +9,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 export function EndpointPolicyPageProvider({ getService, getPageObjects }: FtrProviderContext) {
   const pageObjects = getPageObjects(['common', 'header']);
   const testSubjects = getService('testSubjects');
+  const browser = getService('browser');
 
   return {
     /**
@@ -88,6 +89,7 @@ export function EndpointPolicyPageProvider({ getService, getPageObjects }: FtrPr
      */
     async confirmAndSave() {
       await this.ensureIsOnDetailsPage();
+      await browser.scrollTop();
       await (await this.findSaveButton()).click();
       await testSubjects.existOrFail('policyDetailsConfirmModal');
       await pageObjects.common.clickConfirmOnModal();

From 82ceb87475a1fb19cb0d0271d57e1882235328a1 Mon Sep 17 00:00:00 2001
From: Michael Olorunnisola <michael.olorunnisola@elastic.co>
Date: Fri, 25 Sep 2020 08:33:24 -0400
Subject: [PATCH 09/22] [Security Solution][Resolver] Update @timestamp
 formatting (#78166)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../public/resolver/mocks/get_ui_settings.ts  |  14 +++
 .../public/resolver/mocks/resolver_tree.ts    |  40 +++----
 .../test_utilities/simulator/index.tsx        |  11 +-
 .../public/resolver/view/panel.test.tsx       |   3 +
 .../resolver/view/panels/event_detail.tsx     |  16 +--
 .../resolver/view/panels/node_detail.tsx      |  11 +-
 .../view/panels/node_events_of_type.tsx       |   5 +-
 .../public/resolver/view/panels/node_list.tsx |  22 ++--
 .../view/panels/panel_content_utilities.tsx   |  43 ++-----
 .../view/panels/use_formatted_date.test.tsx   | 106 ++++++++++++++++++
 .../view/panels/use_formatted_date.ts         |  53 +++++++++
 .../translations/translations/ja-JP.json      |   1 -
 .../translations/translations/zh-CN.json      |   1 -
 13 files changed, 240 insertions(+), 86 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts
 create mode 100644 x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx
 create mode 100644 x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.ts

diff --git a/x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts b/x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts
new file mode 100644
index 00000000000000..ab1a5c86859ac5
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/resolver/mocks/get_ui_settings.ts
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export function getUiSettings(key: string): string | undefined {
+  if (key === 'dateFormat') {
+    return 'MMM D, YYYY @ HH:mm:ss.SSS';
+  }
+  if (key === 'dateFormat:tz') {
+    return 'America/New_York';
+  }
+}
diff --git a/x-pack/plugins/security_solution/public/resolver/mocks/resolver_tree.ts b/x-pack/plugins/security_solution/public/resolver/mocks/resolver_tree.ts
index 8691ecac4d1ccf..3f7c58efc762b9 100644
--- a/x-pack/plugins/security_solution/public/resolver/mocks/resolver_tree.ts
+++ b/x-pack/plugins/security_solution/public/resolver/mocks/resolver_tree.ts
@@ -21,19 +21,19 @@ export function mockTreeWith2AncestorsAndNoChildren({
     entityID: secondAncestorID,
     processName: 'a',
     parentEntityID: 'none',
-    timestamp: 0,
+    timestamp: 1600863932316,
   });
   const firstAncestor: SafeResolverEvent = mockEndpointEvent({
     entityID: firstAncestorID,
     processName: 'b',
     parentEntityID: secondAncestorID,
-    timestamp: 1,
+    timestamp: 1600863932317,
   });
   const originEvent: SafeResolverEvent = mockEndpointEvent({
     entityID: originID,
     processName: 'c',
     parentEntityID: firstAncestorID,
-    timestamp: 2,
+    timestamp: 1600863932318,
   });
   return {
     entityID: originID,
@@ -68,39 +68,39 @@ export function mockTreeWithAllProcessesTerminated({
     entityID: secondAncestorID,
     processName: 'a',
     parentEntityID: 'none',
-    timestamp: 0,
+    timestamp: 1600863932316,
   });
   const firstAncestor: SafeResolverEvent = mockEndpointEvent({
     entityID: firstAncestorID,
     processName: 'b',
     parentEntityID: secondAncestorID,
-    timestamp: 1,
+    timestamp: 1600863932317,
   });
   const originEvent: SafeResolverEvent = mockEndpointEvent({
     entityID: originID,
     processName: 'c',
     parentEntityID: firstAncestorID,
-    timestamp: 2,
+    timestamp: 1600863932318,
   });
   const secondAncestorTermination: SafeResolverEvent = mockEndpointEvent({
     entityID: secondAncestorID,
     processName: 'a',
     parentEntityID: 'none',
-    timestamp: 0,
+    timestamp: 1600863932316,
     eventType: 'end',
   });
   const firstAncestorTermination: SafeResolverEvent = mockEndpointEvent({
     entityID: firstAncestorID,
     processName: 'b',
     parentEntityID: secondAncestorID,
-    timestamp: 1,
+    timestamp: 1600863932317,
     eventType: 'end',
   });
   const originEventTermination: SafeResolverEvent = mockEndpointEvent({
     entityID: originID,
     processName: 'c',
     parentEntityID: firstAncestorID,
-    timestamp: 2,
+    timestamp: 1600863932318,
     eventType: 'end',
   });
   return ({
@@ -162,21 +162,21 @@ export function mockTreeWithNoAncestorsAnd2Children({
     entityID: originID,
     processName: 'c.ext',
     parentEntityID: 'none',
-    timestamp: 0,
+    timestamp: 1600863932316,
   });
   const firstChild: SafeResolverEvent = mockEndpointEvent({
     pid: 1,
     entityID: firstChildID,
     processName: 'd',
     parentEntityID: originID,
-    timestamp: 1,
+    timestamp: 1600863932317,
   });
   const secondChild: SafeResolverEvent = mockEndpointEvent({
     pid: 2,
     entityID: secondChildID,
     processName: 'e',
     parentEntityID: originID,
-    timestamp: 2,
+    timestamp: 1600863932318,
   });
 
   return {
@@ -216,50 +216,50 @@ export function mockTreeWith1AncestorAnd2ChildrenAndAllNodesHave2GraphableEvents
   const ancestor: SafeResolverEvent = mockEndpointEvent({
     entityID: ancestorID,
     processName: ancestorID,
-    timestamp: 1,
+    timestamp: 1600863932317,
     parentEntityID: undefined,
   });
   const ancestorClone: SafeResolverEvent = mockEndpointEvent({
     entityID: ancestorID,
     processName: ancestorID,
-    timestamp: 1,
+    timestamp: 1600863932317,
     parentEntityID: undefined,
   });
   const origin: SafeResolverEvent = mockEndpointEvent({
     entityID: originID,
     processName: originID,
     parentEntityID: ancestorID,
-    timestamp: 0,
+    timestamp: 1600863932316,
   });
   const originClone: SafeResolverEvent = mockEndpointEvent({
     entityID: originID,
     processName: originID,
     parentEntityID: ancestorID,
-    timestamp: 0,
+    timestamp: 1600863932316,
   });
   const firstChild: SafeResolverEvent = mockEndpointEvent({
     entityID: firstChildID,
     processName: firstChildID,
     parentEntityID: originID,
-    timestamp: 1,
+    timestamp: 1600863932317,
   });
   const firstChildClone: SafeResolverEvent = mockEndpointEvent({
     entityID: firstChildID,
     processName: firstChildID,
     parentEntityID: originID,
-    timestamp: 1,
+    timestamp: 1600863932317,
   });
   const secondChild: SafeResolverEvent = mockEndpointEvent({
     entityID: secondChildID,
     processName: secondChildID,
     parentEntityID: originID,
-    timestamp: 2,
+    timestamp: 1600863932318,
   });
   const secondChildClone: SafeResolverEvent = mockEndpointEvent({
     entityID: secondChildID,
     processName: secondChildID,
     parentEntityID: originID,
-    timestamp: 2,
+    timestamp: 1600863932318,
   });
 
   return ({
diff --git a/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx b/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx
index 9d10d1c2b64a77..ea603f25834313 100644
--- a/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/test_utilities/simulator/index.tsx
@@ -8,7 +8,6 @@ import React from 'react';
 import { Store, createStore, applyMiddleware } from 'redux';
 import { mount, ReactWrapper } from 'enzyme';
 import { History as HistoryPackageHistoryInterface, createMemoryHistory } from 'history';
-import { CoreStart } from '../../../../../../../src/core/public';
 import { coreMock } from '../../../../../../../src/core/public/mocks';
 import { spyMiddlewareFactory } from '../spy_middleware_factory';
 import { resolverMiddlewareFactory } from '../../store/middleware';
@@ -17,6 +16,7 @@ import { MockResolver } from './mock_resolver';
 import { ResolverState, DataAccessLayer, SpyMiddleware, SideEffectSimulator } from '../../types';
 import { ResolverAction } from '../../store/actions';
 import { sideEffectSimulatorFactory } from '../../view/side_effect_simulator_factory';
+import { getUiSettings } from '../../mocks/get_ui_settings';
 
 /**
  * Test a Resolver instance using jest, enzyme, and a mock data layer.
@@ -91,7 +91,9 @@ export class Simulator {
     this.history = history ?? createMemoryHistory();
 
     // Used for `KibanaContextProvider`
-    const coreStart: CoreStart = coreMock.createStart();
+    const coreStart = coreMock.createStart();
+
+    coreStart.uiSettings.get.mockImplementation(getUiSettings);
 
     this.sideEffectSimulator = sideEffectSimulatorFactory();
 
@@ -296,10 +298,7 @@ export class Simulator {
       const title = titles.at(index).text();
       const description = descriptions.at(index).text();
 
-      // Exclude timestamp since we can't currently calculate the expected description for it from tests
-      if (title !== '@timestamp') {
-        entries.push([title, description]);
-      }
+      entries.push([title, description]);
     }
     return entries;
   }
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx
index 2f23469606acab..63a70716b2d417 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panel.test.tsx
@@ -88,6 +88,7 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and
         title: 'c.ext',
         titleIcon: 'Running Process',
         detailEntries: [
+          ['@timestamp', 'Sep 23, 2020 @ 08:25:32.316'],
           ['process.executable', 'executable'],
           ['process.pid', '0'],
           ['user.name', 'user.name'],
@@ -128,6 +129,7 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and
       await expect(
         simulator().map(() => simulator().nodeDetailDescriptionListEntries())
       ).toYieldEqualTo([
+        ['@timestamp', 'Sep 23, 2020 @ 08:25:32.317'],
         ['process.executable', 'executable'],
         ['process.pid', '1'],
         ['user.name', 'user.name'],
@@ -168,6 +170,7 @@ describe(`Resolver: when analyzing a tree with no ancestors and two children and
       await expect(
         simulator().map(() => simulator().nodeDetailDescriptionListEntries())
       ).toYieldEqualTo([
+        ['@timestamp', 'Sep 23, 2020 @ 08:25:32.316'],
         ['process.executable', 'executable'],
         ['process.pid', '0'],
         ['user.name', 'user.name'],
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx
index 72f0d54d51fa3b..168752c507d5a9 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/event_detail.tsx
@@ -15,7 +15,12 @@ import styled from 'styled-components';
 import { useSelector } from 'react-redux';
 import { FormattedMessage } from 'react-intl';
 import { StyledPanel } from '../styles';
-import { BoldCode, StyledTime, GeneratedText, formatDate } from './panel_content_utilities';
+import {
+  BoldCode,
+  StyledTime,
+  GeneratedText,
+  noTimestampRetrievedText,
+} from './panel_content_utilities';
 import { Breadcrumbs } from './breadcrumbs';
 import * as eventModel from '../../../../common/endpoint/models/event';
 import * as selectors from '../../store/selectors';
@@ -25,6 +30,7 @@ import { DescriptiveName } from './descriptive_name';
 import { useLinkProps } from '../use_link_props';
 import { SafeResolverEvent } from '../../../../common/endpoint/types';
 import { deepObjectEntries } from './deep_object_entries';
+import { useFormattedDate } from './use_formatted_date';
 
 export const EventDetail = memo(function EventDetail({
   nodeID,
@@ -78,12 +84,8 @@ const EventDetailContents = memo(function ({
   eventType: string;
   processEvent: SafeResolverEvent;
 }) {
-  const formattedDate = useMemo(() => {
-    const timestamp = eventModel.timestampSafeVersion(event);
-    if (timestamp !== undefined) {
-      return formatDate(new Date(timestamp));
-    }
-  }, [event]);
+  const timestamp = eventModel.timestampSafeVersion(event);
+  const formattedDate = useFormattedDate(timestamp) || noTimestampRetrievedText;
 
   return (
     <StyledPanel>
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx
index 04e9de61f62568..181c9ac8ab8a08 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_detail.tsx
@@ -16,7 +16,7 @@ import { EuiDescriptionListProps } from '@elastic/eui/src/components/description
 import { StyledDescriptionList, StyledTitle } from './styles';
 import * as selectors from '../../store/selectors';
 import * as eventModel from '../../../../common/endpoint/models/event';
-import { formatDate, GeneratedText } from './panel_content_utilities';
+import { GeneratedText } from './panel_content_utilities';
 import { Breadcrumbs } from './breadcrumbs';
 import { processPath, processPID } from '../../models/process_event';
 import { CubeForProcess } from './cube_for_process';
@@ -26,6 +26,7 @@ import { ResolverState } from '../../types';
 import { PanelLoading } from './panel_loading';
 import { StyledPanel } from '../styles';
 import { useLinkProps } from '../use_link_props';
+import { useFormattedDate } from './use_formatted_date';
 
 const StyledCubeForProcess = styled(CubeForProcess)`
   position: relative;
@@ -65,10 +66,10 @@ const NodeDetailView = memo(function ({
   const relatedEventTotal = useSelector((state: ResolverState) => {
     return selectors.relatedEventTotalCount(state)(nodeID);
   });
-  const processInfoEntry: EuiDescriptionListProps['listItems'] = useMemo(() => {
-    const eventTime = eventModel.eventTimestamp(processEvent);
-    const dateTime = eventTime === undefined ? null : formatDate(eventTime);
+  const eventTime = eventModel.eventTimestamp(processEvent);
+  const dateTime = useFormattedDate(eventTime);
 
+  const processInfoEntry: EuiDescriptionListProps['listItems'] = useMemo(() => {
     const createdEntry = {
       title: '@timestamp',
       description: dateTime,
@@ -131,7 +132,7 @@ const NodeDetailView = memo(function ({
       });
 
     return processDescriptionListData;
-  }, [processEvent]);
+  }, [dateTime, processEvent]);
 
   const nodesLinkNavProps = useLinkProps({
     panelView: 'nodes',
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx
index 281794ac24d244..771a143a9c0cde 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.tsx
@@ -10,7 +10,7 @@ import { EuiSpacer, EuiText, EuiButtonEmpty, EuiHorizontalRule } from '@elastic/
 import { useSelector } from 'react-redux';
 import { FormattedMessage } from 'react-intl';
 import { StyledPanel } from '../styles';
-import { formatDate, BoldCode, StyledTime } from './panel_content_utilities';
+import { BoldCode, noTimestampRetrievedText, StyledTime } from './panel_content_utilities';
 import { Breadcrumbs } from './breadcrumbs';
 import * as eventModel from '../../../../common/endpoint/models/event';
 import { SafeResolverEvent } from '../../../../common/endpoint/types';
@@ -19,6 +19,7 @@ import { ResolverState } from '../../types';
 import { PanelLoading } from './panel_loading';
 import { DescriptiveName } from './descriptive_name';
 import { useLinkProps } from '../use_link_props';
+import { useFormattedDate } from './use_formatted_date';
 
 /**
  * Render a list of events that are related to `nodeID` and that have a category of `eventType`.
@@ -83,7 +84,7 @@ const NodeEventsListItem = memo(function ({
   eventType: string;
 }) {
   const timestamp = eventModel.eventTimestamp(event);
-  const date = timestamp !== undefined ? formatDate(timestamp) : timestamp;
+  const date = useFormattedDate(timestamp) || noTimestampRetrievedText;
   const linkProps = useLinkProps({
     panelView: 'eventDetail',
     panelParameters: {
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx
index 8fc6e7cc66c790..78d3477301539f 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_list.tsx
@@ -32,7 +32,6 @@ import {
 } from './styles';
 import * as eventModel from '../../../../common/endpoint/models/event';
 import * as selectors from '../../store/selectors';
-import { formatter } from './panel_content_utilities';
 import { Breadcrumbs } from './breadcrumbs';
 import { CubeForProcess } from './cube_for_process';
 import { LimitWarning } from '../limit_warnings';
@@ -41,6 +40,8 @@ import { useLinkProps } from '../use_link_props';
 import { useColors } from '../use_colors';
 import { SafeResolverEvent } from '../../../../common/endpoint/types';
 import { ResolverAction } from '../../store/actions';
+import { useFormattedDate } from './use_formatted_date';
+import { getEmptyTagValue } from '../../../common/components/empty_value';
 
 interface ProcessTableView {
   name?: string;
@@ -80,18 +81,7 @@ export const NodeList = memo(() => {
         dataType: 'date',
         sortable: true,
         render(eventDate?: Date) {
-          return eventDate ? (
-            formatter.format(eventDate)
-          ) : (
-            <EuiBadge color="warning">
-              {i18n.translate(
-                'xpack.securitySolution.endpoint.resolver.panel.table.row.timestampInvalidLabel',
-                {
-                  defaultMessage: 'invalid',
-                }
-              )}
-            </EuiBadge>
-          );
+          return <NodeDetailTimestamp eventDate={eventDate} />;
         },
       },
     ],
@@ -220,3 +210,9 @@ function NodeDetailLink({
     </EuiButtonEmpty>
   );
 }
+
+const NodeDetailTimestamp = memo(({ eventDate }: { eventDate: Date | undefined }) => {
+  const formattedDate = useFormattedDate(eventDate);
+
+  return formattedDate ? <>{formattedDate}</> : getEmptyTagValue();
+});
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx
index 5ca34b33b2396e..a20498cbfb67b2 100644
--- a/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/panel_content_utilities.tsx
@@ -6,11 +6,22 @@
 
 /* eslint-disable react/display-name */
 
-import { i18n } from '@kbn/i18n';
 import { EuiCode } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
 import styled from 'styled-components';
 import React, { memo } from 'react';
 
+/**
+ * Text to use in place of an undefined timestamp value
+ */
+
+export const noTimestampRetrievedText = i18n.translate(
+  'xpack.securitySolution.enpdoint.resolver.panelutils.noTimestampRetrieved',
+  {
+    defaultMessage: 'No timestamp retrieved',
+  }
+);
+
 /**
  * A bold version of EuiCode to display certain titles with
  */
@@ -59,33 +70,3 @@ export const StyledTime = memo(styled('time')`
   display: inline-block;
   text-align: start;
 `);
-
-/**
- * Long formatter (to second) for DateTime
- */
-export const formatter = new Intl.DateTimeFormat(i18n.getLocale(), {
-  year: 'numeric',
-  month: '2-digit',
-  day: '2-digit',
-  hour: '2-digit',
-  minute: '2-digit',
-  second: '2-digit',
-});
-
-/**
- * @returns {string} A nicely formatted string for a date
- */
-export function formatDate(
-  /** To be passed through Date->Intl.DateTimeFormat */ timestamp: ConstructorParameters<
-    typeof Date
-  >[0]
-): string {
-  const date = new Date(timestamp);
-  if (isFinite(date.getTime())) {
-    return formatter.format(date);
-  } else {
-    return i18n.translate('xpack.securitySolution.enpdoint.resolver.panelutils.invaliddate', {
-      defaultMessage: 'Invalid Date',
-    });
-  }
-}
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx
new file mode 100644
index 00000000000000..9e9ae26900efaf
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.test.tsx
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { render, RenderResult } from '@testing-library/react';
+import { useFormattedDate } from './use_formatted_date';
+import { coreMock } from '../../../../../../../src/core/public/mocks';
+import { KibanaContextProvider } from '../../../../../../../src/plugins/kibana_react/public';
+import { getUiSettings } from '../../mocks/get_ui_settings';
+
+describe('useFormattedDate', () => {
+  let element: HTMLElement;
+  const testID = 'formattedDate';
+  let reactRenderResult: (
+    date: ConstructorParameters<typeof Date>[0] | Date | undefined
+  ) => RenderResult;
+
+  beforeEach(async () => {
+    const mockCoreStart = coreMock.createStart();
+    mockCoreStart.uiSettings.get.mockImplementation(getUiSettings);
+
+    function Test({ date }: { date: ConstructorParameters<typeof Date>[0] | Date | undefined }) {
+      const formattedDate = useFormattedDate(date);
+      return <div data-test-subj={testID}>{formattedDate}</div>;
+    }
+
+    reactRenderResult = (
+      date: ConstructorParameters<typeof Date>[0] | Date | undefined
+    ): RenderResult =>
+      render(
+        <KibanaContextProvider services={mockCoreStart}>
+          <Test date={date} />
+        </KibanaContextProvider>
+      );
+  });
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('when the provided date is undefined', () => {
+    it('should return undefined', async () => {
+      const { findByTestId } = reactRenderResult(undefined);
+      element = await findByTestId(testID);
+
+      expect(element).toBeEmpty();
+    });
+  });
+
+  describe('when the provided date is empty', () => {
+    it('should return undefined', async () => {
+      const { findByTestId } = reactRenderResult('');
+      element = await findByTestId(testID);
+
+      expect(element).toBeEmpty();
+    });
+  });
+
+  describe('when the provided date is an invalid date', () => {
+    it('should return the string invalid date', async () => {
+      const { findByTestId } = reactRenderResult('randomString');
+      element = await findByTestId(testID);
+
+      expect(element).toHaveTextContent('Invalid Date');
+    });
+  });
+
+  describe('when the provided date is a stringified unix timestamp', () => {
+    it('should return the string invalid date', async () => {
+      const { findByTestId } = reactRenderResult('1600863932316');
+      element = await findByTestId(testID);
+
+      expect(element).toHaveTextContent('Invalid Date');
+    });
+  });
+
+  describe('when the provided date is a valid numerical timestamp', () => {
+    it('should return the string invalid date', async () => {
+      const { findByTestId } = reactRenderResult(1600863932316);
+      element = await findByTestId(testID);
+
+      expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.316');
+    });
+  });
+
+  describe('when the provided date is a date string', () => {
+    it('should return the string invalid date', async () => {
+      const { findByTestId } = reactRenderResult('2020-09-23T12:25:32Z');
+      element = await findByTestId(testID);
+
+      expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.000');
+    });
+  });
+
+  describe('when the provided date is a valid date', () => {
+    it('should return the string invalid date', async () => {
+      const validDate = new Date(1600863932316);
+      const { findByTestId } = reactRenderResult(validDate);
+      element = await findByTestId(testID);
+
+      expect(element).toHaveTextContent('Sep 23, 2020 @ 08:25:32.316');
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.ts b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.ts
new file mode 100644
index 00000000000000..05e7154dd6fdd3
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/resolver/view/panels/use_formatted_date.ts
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { i18n } from '@kbn/i18n';
+import moment from 'moment-timezone';
+import { useUiSetting } from '../../../../../../../src/plugins/kibana_react/public';
+
+const invalidDateText = i18n.translate(
+  'xpack.securitySolution.enpdoint.resolver.panelutils.invaliddate',
+  {
+    defaultMessage: 'Invalid Date',
+  }
+);
+
+/**
+ * Long formatter (to second) for DateTime
+ */
+const formatter = new Intl.DateTimeFormat(i18n.getLocale(), {
+  year: 'numeric',
+  month: '2-digit',
+  day: '2-digit',
+  hour: '2-digit',
+  minute: '2-digit',
+  second: '2-digit',
+});
+
+/**
+ *
+ * @description formats a given time based on the user defined format in the advanced settings section of kibana under dateFormat
+ * @export
+ * @param {(ConstructorParameters<typeof Date>[0] | undefined)} timestamp
+ * @returns {(string | null)} - Either a formatted date or the text 'Invalid Date'
+ */
+export function useFormattedDate(
+  timestamp: ConstructorParameters<typeof Date>[0] | Date | undefined
+): string | undefined {
+  const dateFormatSetting: string = useUiSetting('dateFormat');
+  const timezoneSetting: string = useUiSetting('dateFormat:tz');
+  const usableTimezoneSetting = timezoneSetting === 'Browser' ? moment.tz.guess() : timezoneSetting;
+
+  if (!timestamp) return undefined;
+
+  const date = new Date(timestamp);
+  if (date && Number.isFinite(date.getTime())) {
+    return dateFormatSetting
+      ? moment.tz(date, usableTimezoneSetting).format(dateFormatSetting)
+      : formatter.format(date);
+  }
+
+  return invalidDateText;
+}
diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json
index 505affb4f419aa..fd743400133a7d 100644
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@@ -15883,7 +15883,6 @@
     "xpack.securitySolution.endpoint.resolver.panel.table.row.count": "カウント",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.eventType": "イベントタイプ",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.processNameTitle": "プロセス名",
-    "xpack.securitySolution.endpoint.resolver.panel.table.row.timestampInvalidLabel": "無効",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.timestampTitle": "タイムスタンプ",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.valueMissingDescription": "値が見つかりません",
     "xpack.securitySolution.endpoint.resolver.relatedEventLimitExceeded": "{numberOfEventsMissing} {category}件のイベントを表示できませんでした。データの上限に達しました。",
diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json
index 3bca7ea36661d7..104fc70f5dd715 100644
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@@ -15893,7 +15893,6 @@
     "xpack.securitySolution.endpoint.resolver.panel.table.row.count": "计数",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.eventType": "事件类型",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.processNameTitle": "进程名称",
-    "xpack.securitySolution.endpoint.resolver.panel.table.row.timestampInvalidLabel": "无效",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.timestampTitle": "时间戳",
     "xpack.securitySolution.endpoint.resolver.panel.table.row.valueMissingDescription": "值缺失",
     "xpack.securitySolution.endpoint.resolver.relatedEventLimitExceeded": "{numberOfEventsMissing} 个{category}事件无法显示，因为已达到数据限制。",

From f2fc48dec8bb930fd8f8d138fa502f0a7d1ca944 Mon Sep 17 00:00:00 2001
From: Christos Nasikas <christos.nasikas@elastic.co>
Date: Fri, 25 Sep 2020 16:45:19 +0300
Subject: [PATCH 10/22] [Docs][Actions] Add docs for Jira and IBM Resilient
 (#78316)

---
 docs/user/alerting/action-types.asciidoc      | 13 +++-
 docs/user/alerting/action-types/jira.asciidoc | 77 +++++++++++++++++++
 .../alerting/action-types/pagerduty.asciidoc  |  2 +-
 .../alerting/action-types/resilient.asciidoc  | 76 ++++++++++++++++++
 .../alerting/action-types/servicenow.asciidoc |  8 +-
 .../alerting-getting-started.asciidoc         |  6 +-
 docs/user/alerting/defining-alerts.asciidoc   |  2 +-
 7 files changed, 174 insertions(+), 10 deletions(-)
 create mode 100644 docs/user/alerting/action-types/jira.asciidoc
 create mode 100644 docs/user/alerting/action-types/resilient.asciidoc

diff --git a/docs/user/alerting/action-types.asciidoc b/docs/user/alerting/action-types.asciidoc
index be31458ff39fae..af80b17f8605f7 100644
--- a/docs/user/alerting/action-types.asciidoc
+++ b/docs/user/alerting/action-types.asciidoc
@@ -11,10 +11,19 @@ a| <<email-action-type, Email>>
 
 | Send email from your server.
 
+a| <<resilient-action-type, IBM Resilient>>
+
+| Create an incident in IBM Resilient.
+
 a| <<index-action-type, Index>>
 
 | Index data into Elasticsearch.
 
+a| <<jira-action-type, Jira>>
+
+| Create an incident in Jira.
+
+
 a| <<pagerduty-action-type, PagerDuty>>
 
 | Send an event in PagerDuty.
@@ -53,10 +62,12 @@ before {kib} starts.
 If you preconfigure a connector, you can also <<preconfigured-action-type-example, preconfigure its action type>>.
 
 include::action-types/email.asciidoc[]
+include::action-types/resilient.asciidoc[]
 include::action-types/index.asciidoc[]
+include::action-types/jira.asciidoc[]
 include::action-types/pagerduty.asciidoc[]
 include::action-types/server-log.asciidoc[]
+include::action-types/servicenow.asciidoc[]
 include::action-types/slack.asciidoc[]
 include::action-types/webhook.asciidoc[]
 include::action-types/pre-configured-connectors.asciidoc[]
-include::action-types/servicenow.asciidoc[]
diff --git a/docs/user/alerting/action-types/jira.asciidoc b/docs/user/alerting/action-types/jira.asciidoc
new file mode 100644
index 00000000000000..48bd6c8501b9f6
--- /dev/null
+++ b/docs/user/alerting/action-types/jira.asciidoc
@@ -0,0 +1,77 @@
+[role="xpack"]
+[[jira-action-type]]
+=== Jira action
+
+The Jira action type uses the https://developer.atlassian.com/cloud/jira/platform/rest/v2/[REST API v2] to create Jira issues.
+
+[float]
+[[jira-connector-configuration]]
+==== Connector configuration
+
+Jira connectors have the following configuration properties:
+
+Name::      The name of the connector. The name is used to identify a  connector in the **Stack Management** UI connector listing, and in the connector list when configuring an action.
+URL::       Jira instance URL.
+Project key:: Jira project key.
+Email (or username)::  The account email (or username) for HTTP Basic authentication.
+API token (or password)::  Jira API authentication token (or password) for HTTP Basic authentication.
+
+[float]
+[[Preconfigured-jira-configuration]]
+==== Preconfigured action type
+
+[source,text]
+--
+ my-jira:
+   name: preconfigured-jira-action-type
+   actionTypeId: .jira
+   config:
+     apiUrl: https://elastic.atlassian.net
+     projectKey: ES
+   secrets:
+     email: testuser
+     apiToken: tokenkeystorevalue
+--
+
+`config` defines the action type specific to the configuration and contains the following properties:
+
+[cols="2*<"]
+|===
+
+| `apiUrl`
+| An address that corresponds to *URL*.
+
+| `projectKey`
+| A key that corresponds to *Project Key*.
+
+|===
+
+`secrets` defines sensitive information for the action type:
+
+[cols="2*<"]
+|===
+
+| `email`
+| A string that corresponds to *Email*.
+
+| `apiToken`
+| A string that corresponds to *API Token*. Should be stored in the <<creating-keystore, {kib} keystore>>.
+
+|===
+
+[[jira-action-configuration]]
+==== Action configuration
+
+Jira actions have the following configuration properties:
+
+Issue type::              The type of the issue.
+Priority::              The priority of the incident.
+Labels::              The labels of the incident.
+Title::    A title for the issue, used for searching the contents of the knowledge base. 
+Description::          The details about the incident.
+Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
+
+[[configuring-jira]]
+==== Configuring and testing Jira
+
+Jira offers free https://www.atlassian.com/software/jira/free[Instances], which you can use to test incidents.
diff --git a/docs/user/alerting/action-types/pagerduty.asciidoc b/docs/user/alerting/action-types/pagerduty.asciidoc
index 2c9add5233c913..9301224e6df48e 100644
--- a/docs/user/alerting/action-types/pagerduty.asciidoc
+++ b/docs/user/alerting/action-types/pagerduty.asciidoc
@@ -36,7 +36,7 @@ This is required to encrypt parameters that must be secured, for example PagerDu
 If you have security enabled:
 
 * You must have
-application privileges to access Metrics, APM, Uptime, or SIEM.
+application privileges to access Metrics, APM, Uptime, or Security.
 * If you are using a self-managed deployment with security, you must have
 Transport Security Layer (TLS) enabled for communication <<configuring-tls-kib-es, between Elasticsearch and Kibana>>.
 Alerts uses API keys to secure background alert checks and actions,
diff --git a/docs/user/alerting/action-types/resilient.asciidoc b/docs/user/alerting/action-types/resilient.asciidoc
new file mode 100644
index 00000000000000..b5ddb76d49b0cd
--- /dev/null
+++ b/docs/user/alerting/action-types/resilient.asciidoc
@@ -0,0 +1,76 @@
+[role="xpack"]
+[[resilient-action-type]]
+=== IBM Resilient action
+
+The IBM Resilient action type uses the https://developer.ibm.com/security/resilient/rest/[RESILIENT REST v2] to create IBM Resilient incidents.
+
+[float]
+[[resilient-connector-configuration]]
+==== Connector configuration
+
+IBM Resilient connectors have the following configuration properties:
+
+Name::      The name of the connector. The name is used to identify a  connector in the **Stack Management** UI connector listing, and in the connector list when configuring an action.
+URL::       IBM Resilient instance URL.
+Organization ID:: IBM Resilient organization ID.
+API key ID::  The authentication key ID for HTTP Basic authentication.
+API key secret::  The authentication key secret for HTTP Basic authentication.
+
+[float]
+[[Preconfigured-resilient-configuration]]
+==== Preconfigured action type
+
+[source,text]
+--
+ my-resilient:
+   name: preconfigured-resilient-action-type
+   actionTypeId: .resilient
+   config:
+     apiUrl: https://elastic.resilient.net
+     orgId: ES
+   secrets:
+     apiKeyId: testuser
+     apiKeySecret: tokenkeystorevalue
+--
+
+`config` defines the action type specific to the configuration and contains the following properties:
+
+[cols="2*<"]
+|===
+
+| `apiUrl`
+| An address that corresponds to *URL*.
+
+| `orgId`
+| An ID that corresponds to *Organization ID*.
+
+|===
+
+`secrets` defines sensitive information for the action type:
+
+[cols="2*<"]
+|===
+
+| `apiKeyId`
+| A string that corresponds to *API key ID*.
+
+| `apiKeySecret`
+| A string that corresponds to *API Key secret*. Should be stored in the <<creating-keystore, {kib} keystore>>.
+
+|===
+
+[[resilient-action-configuration]]
+==== Action configuration
+
+IBM Resilient actions have the following configuration properties:
+
+Incident types::              The incident types of the incident.
+Severity code::              The severity of the incident.
+Name::    A name for the issue, used for searching the contents of the knowledge base. 
+Description::          The details about the incident.
+Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
+
+[[configuring-resilient]]
+==== Configuring and testing IBM Resilient
+
+IBM Resilient offers https://www.ibm.com/security/intelligent-orchestration/resilient[Instances], which you can use to test incidents.
diff --git a/docs/user/alerting/action-types/servicenow.asciidoc b/docs/user/alerting/action-types/servicenow.asciidoc
index 32f828aea2357e..0acb92bcdb5ee5 100644
--- a/docs/user/alerting/action-types/servicenow.asciidoc
+++ b/docs/user/alerting/action-types/servicenow.asciidoc
@@ -10,7 +10,7 @@ The ServiceNow action type uses the https://developer.servicenow.com/app.do#!/re
 
 ServiceNow connectors have the following configuration properties:
 
-Name::      The name of the connector. The name is used to identify a  connector in the management UI connector listing, or in the connector list when configuring an action.
+Name::      The name of the connector. The name is used to identify a  connector in the **Stack Management** UI connector listing, and in the connector list when configuring an action.
 URL::       ServiceNow instance URL.
 Username::  Username for HTTP Basic authentication.
 Password::  Password for HTTP Basic authentication.
@@ -37,7 +37,7 @@ Password::  Password for HTTP Basic authentication.
 |===
 
 | `apiUrl`
-| An address that corresponds to *Sender*.
+| An address that corresponds to *URL*.
 
 |===
 
@@ -47,7 +47,7 @@ Password::  Password for HTTP Basic authentication.
 |===
 
 | `username`
-| A string that corresponds to *User*.
+| A string that corresponds to *Username*.
 
 | `password`
 | A string that corresponds to *Password*. Should be stored in the <<creating-keystore, {kib} keystore>>.
@@ -62,7 +62,7 @@ ServiceNow actions have the following configuration properties:
 Urgency::              The extent to which the incident resolution can delay.
 Severity::             The severity of the incident.
 Impact::               The effect an incident has on business. Can be measured by the number of affected users or by how critical it is to the business in question.
-Short description::    A short description of the incident, used for searching the contents of the knowledge base. 
+Short description::    A short description for the incident, used for searching the contents of the knowledge base. 
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
 
diff --git a/docs/user/alerting/alerting-getting-started.asciidoc b/docs/user/alerting/alerting-getting-started.asciidoc
index 6bc085b0f78b9e..bdb72b1658cd26 100644
--- a/docs/user/alerting/alerting-getting-started.asciidoc
+++ b/docs/user/alerting/alerting-getting-started.asciidoc
@@ -6,7 +6,7 @@ beta[]
 
 --
 
-Alerting allows you to detect complex conditions within different {kib} apps and trigger actions when those conditions are met. Alerting is integrated with <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*SIEM*>>, <<xpack-uptime,*Uptime*>>, can be centrally managed from the <<management,*Management*>> UI, and provides a set of built-in <<action-types, actions>> and <<alert-types, alerts>> for you to use.
+Alerting allows you to detect complex conditions within different {kib} apps and trigger actions when those conditions are met. Alerting is integrated with <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*Security*>>, <<xpack-uptime,*Uptime*>>, can be centrally managed from the <<management,*Management*>> UI, and provides a set of built-in <<action-types, actions>> and <<alert-types, alerts>> for you to use.
 
 image::images/alerting-overview.png[Alerts and actions UI]
 
@@ -148,7 +148,7 @@ Functionally, {kib} alerting differs in that:
 * {kib} alerts tracks and persists the state of each detected condition through *alert instances*. This makes it possible to mute and throttle individual instances, and detect changes in state such as resolution. 
 * Actions are linked to *alert instances* in {kib} alerting. Actions are fired for each occurrence of a detected condition, rather than for the entire alert. 
 
-At a higher level, {kib} alerts allow rich integrations across use cases like <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*SIEM*>>, and <<xpack-uptime,*Uptime*>>.
+At a higher level, {kib} alerts allow rich integrations across use cases like <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*Security*>>, and <<xpack-uptime,*Uptime*>>.
 Pre-packaged *alert types* simplify setup, hide the details complex domain-specific detections, while providing a consistent interface across {kib}.
 
 [float]
@@ -171,7 +171,7 @@ To access alerting in a space, a user must have access to one of the following f
 
 * <<xpack-apm,*APM*>>
 * <<xpack-infra,*Metrics*>>
-* <<xpack-siem,*SIEM*>>
+* <<xpack-siem,*Security*>>
 * <<xpack-uptime,*Uptime*>>
 
 See <<kibana-feature-privileges, feature privileges>> for more information on configuring roles that provide access to these features. 
diff --git a/docs/user/alerting/defining-alerts.asciidoc b/docs/user/alerting/defining-alerts.asciidoc
index d05a727016455f..7f201d2c39e89c 100644
--- a/docs/user/alerting/defining-alerts.asciidoc
+++ b/docs/user/alerting/defining-alerts.asciidoc
@@ -2,7 +2,7 @@
 [[defining-alerts]]
 == Defining alerts
 
-{kib} alerts can be created in a variety of apps including <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*SIEM*>>, <<xpack-uptime,*Uptime*>> and from <<management,*Management*>> UI. While alerting details may differ from app to app, they share a common interface for defining and configuring alerts that this section describes in more detail.
+{kib} alerts can be created in a variety of apps including <<xpack-apm,*APM*>>, <<xpack-infra,*Metrics*>>, <<xpack-siem,*Security*>>, <<xpack-uptime,*Uptime*>> and from <<management,*Management*>> UI. While alerting details may differ from app to app, they share a common interface for defining and configuring alerts that this section describes in more detail.
 
 [float]
 === Alert flyout

From a88c27258e9ad91512ce50795be4d5c9626d3b24 Mon Sep 17 00:00:00 2001
From: Chris Roberson <chrisronline@gmail.com>
Date: Fri, 25 Sep 2020 10:19:53 -0400
Subject: [PATCH 11/22] [Monitoring] Usage collection (#75878)

* First stab at some internal telemetry

* Add missing files

* mbCount telemetry

* Include more data

* Remove unused field

* This file isn't used

* Mock in tests

* Add schema

* Store schema

* Use sample cluster instead

* Fix telemetry schema

* Fix type issues

* Updates

* Fix schema and tests

* Add tests

* Add tests

* Go back to using an array

* Fix schema

* Add page view data

* Remove debug

* Handle loading scenario here

* Add delay tracking too

* Add clicks for setup mode

* Add clicks for setup mode

* Fix beats/apm page views

* Fix typings
---
 x-pack/plugins/monitoring/common/constants.ts |   7 +
 x-pack/plugins/monitoring/kibana.json         |   3 +-
 .../monitoring/public/angular/index.ts        |   2 +
 .../public/components/page_loading/index.js   |  18 +-
 .../components/setup_mode/enter_button.tsx    |   7 +
 .../plugins/monitoring/public/legacy_shims.ts |   5 +-
 .../monitoring/public/lib/setup_mode.tsx      |  12 +-
 x-pack/plugins/monitoring/public/plugin.ts    |   3 +
 x-pack/plugins/monitoring/public/types.ts     |   2 +
 .../public/views/apm/instance/index.js        |  22 +--
 .../public/views/apm/instances/index.js       |  54 +++---
 .../public/views/apm/overview/index.js        |   9 +-
 .../public/views/base_controller.js           |  20 ++-
 .../public/views/beats/beat/index.js          |   1 +
 .../public/views/beats/listing/index.js       |   4 +-
 .../public/views/cluster/listing/index.js     |   1 +
 .../public/views/cluster/overview/index.js    |   1 +
 .../public/views/elasticsearch/ccr/index.js   |   9 +-
 .../views/elasticsearch/ccr/shard/index.js    |  11 +-
 .../elasticsearch/index/advanced/index.js     |   1 +
 .../public/views/elasticsearch/index/index.js |   1 +
 .../views/elasticsearch/indices/index.js      |  31 ++--
 .../elasticsearch/node/advanced/index.js      |   1 +
 .../public/views/elasticsearch/node/index.js  |   7 +
 .../public/views/elasticsearch/nodes/index.js |  63 +++----
 .../elasticsearch/overview/controller.js      |   2 +-
 .../public/views/kibana/instance/index.js     |   2 +-
 .../views/logstash/node/advanced/index.js     |   1 +
 .../public/views/logstash/node/index.js       |   1 +
 .../views/logstash/node/pipelines/index.js    |   1 +
 .../collectors/get_usage_collector.test.ts    | 161 ++++++++++++++++++
 .../collectors/get_usage_collector.ts         | 128 ++++++++++++++
 .../kibana_monitoring/collectors/index.ts     |   8 +-
 .../collectors/lib/fetch_es_usage.test.ts     |  97 +++++++++++
 .../collectors/lib/fetch_es_usage.ts          | 122 +++++++++++++
 .../collectors/lib/fetch_license_type.test.ts |  40 +++++
 .../collectors/lib/fetch_license_type.ts      |  57 +++++++
 .../lib/fetch_stack_product_usage.test.ts     | 117 +++++++++++++
 .../lib/fetch_stack_product_usage.ts          | 111 ++++++++++++
 .../lib/get_stack_products_usage.test.ts      |  31 ++++
 .../lib/get_stack_products_usage.ts           |  77 +++++++++
 .../kibana_monitoring/collectors/types.ts     |  27 +++
 .../lib/alerts/get_ccs_index_pattern.ts       |   3 +
 .../plugins/monitoring/server/plugin.test.ts  |  11 +-
 x-pack/plugins/monitoring/server/plugin.ts    |  16 +-
 .../typings/fetch_overview_data/index.ts      |   5 +-
 .../plugins/observability/typings/common.ts   |   8 +-
 .../schema/xpack_plugins.json                 |  85 +++++++++
 48 files changed, 1287 insertions(+), 119 deletions(-)
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.test.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.test.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.test.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.test.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.test.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.ts
 create mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/types.ts

diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts
index 2c714080969e46..8be0eb0b06823a 100644
--- a/x-pack/plugins/monitoring/common/constants.ts
+++ b/x-pack/plugins/monitoring/common/constants.ts
@@ -273,3 +273,10 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email';
 export const ALERT_ACTION_TYPE_LOG = '.server-log';
 
 export const ALERT_EMAIL_SERVICES = ['gmail', 'hotmail', 'icloud', 'outlook365', 'ses', 'yahoo'];
+
+/**
+ * The saved object type for various monitoring data
+ */
+export const SAVED_OBJECT_TELEMETRY = 'monitoring-telemetry';
+
+export const TELEMETRY_METRIC_BUTTON_CLICK = 'btnclick__';
diff --git a/x-pack/plugins/monitoring/kibana.json b/x-pack/plugins/monitoring/kibana.json
index 2b8756ea0cb46d..8b0b0b7aae6931 100644
--- a/x-pack/plugins/monitoring/kibana.json
+++ b/x-pack/plugins/monitoring/kibana.json
@@ -12,7 +12,8 @@
     "triggers_actions_ui",
     "alerts",
     "actions",
-    "encryptedSavedObjects"
+    "encryptedSavedObjects",
+    "observability"
   ],
   "optionalPlugins": ["infra", "telemetryCollectionManager", "usageCollection", "home", "cloud"],
   "server": true,
diff --git a/x-pack/plugins/monitoring/public/angular/index.ts b/x-pack/plugins/monitoring/public/angular/index.ts
index da57c028643a54..3c30d3c358a145 100644
--- a/x-pack/plugins/monitoring/public/angular/index.ts
+++ b/x-pack/plugins/monitoring/public/angular/index.ts
@@ -26,6 +26,7 @@ export class AngularApp {
       pluginInitializerContext,
       externalConfig,
       triggersActionsUi,
+      usageCollection,
       kibanaLegacy,
     } = deps;
     const app: IModule = localAppModule(deps);
@@ -42,6 +43,7 @@ export class AngularApp {
           externalConfig,
           kibanaLegacy,
           triggersActionsUi,
+          usageCollection,
         },
         this.injector
       );
diff --git a/x-pack/plugins/monitoring/public/components/page_loading/index.js b/x-pack/plugins/monitoring/public/components/page_loading/index.js
index c8a0404ec717b4..6af11cd38c378d 100644
--- a/x-pack/plugins/monitoring/public/components/page_loading/index.js
+++ b/x-pack/plugins/monitoring/public/components/page_loading/index.js
@@ -15,8 +15,9 @@ import {
 } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import './page_loading.scss';
+import { useTrackPageview } from '../../../../observability/public';
 
-export function PageLoading() {
+function PageLoadingUI() {
   return (
     <EuiPage style={{ height: 'calc(100vh - 50px)' }}>
       <EuiPageBody>
@@ -45,3 +46,18 @@ export function PageLoading() {
     </EuiPage>
   );
 }
+
+function PageLoadingTracking({ pageViewTitle }) {
+  const path = pageViewTitle.toLowerCase().replace(/-/g, '').replace(/\s+/g, '_');
+  useTrackPageview({ app: 'stack_monitoring', path });
+  useTrackPageview({ app: 'stack_monitoring', path, delay: 15000 });
+  return <PageLoadingUI />;
+}
+
+export function PageLoading({ pageViewTitle }) {
+  if (pageViewTitle) {
+    return <PageLoadingTracking pageViewTitle={pageViewTitle} />;
+  }
+
+  return <PageLoadingUI />;
+}
diff --git a/x-pack/plugins/monitoring/public/components/setup_mode/enter_button.tsx b/x-pack/plugins/monitoring/public/components/setup_mode/enter_button.tsx
index e06113255c1ef0..b47b51e664f5fa 100644
--- a/x-pack/plugins/monitoring/public/components/setup_mode/enter_button.tsx
+++ b/x-pack/plugins/monitoring/public/components/setup_mode/enter_button.tsx
@@ -8,6 +8,8 @@ import React from 'react';
 import { EuiButton } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import './enter_button.scss';
+import { METRIC_TYPE, useUiTracker } from '../../../../observability/public';
+import { TELEMETRY_METRIC_BUTTON_CLICK } from '../../../common/constants';
 
 export interface SetupModeEnterButtonProps {
   enabled: boolean;
@@ -18,6 +20,7 @@ export const SetupModeEnterButton: React.FC<SetupModeEnterButtonProps> = (
   props: SetupModeEnterButtonProps
 ) => {
   const [isLoading, setIsLoading] = React.useState(false);
+  const trackStat = useUiTracker({ app: 'stack_monitoring' });
 
   if (!props.enabled) {
     return null;
@@ -26,6 +29,10 @@ export const SetupModeEnterButton: React.FC<SetupModeEnterButtonProps> = (
   async function enterSetupMode() {
     setIsLoading(true);
     await props.toggleSetupMode(true);
+    trackStat({
+      metric: `${TELEMETRY_METRIC_BUTTON_CLICK}setupmode_enter`,
+      metricType: METRIC_TYPE.CLICK,
+    });
     setIsLoading(false);
   }
 
diff --git a/x-pack/plugins/monitoring/public/legacy_shims.ts b/x-pack/plugins/monitoring/public/legacy_shims.ts
index 0f979e5637d686..488450bafd3a28 100644
--- a/x-pack/plugins/monitoring/public/legacy_shims.ts
+++ b/x-pack/plugins/monitoring/public/legacy_shims.ts
@@ -14,6 +14,7 @@ import { TriggersAndActionsUIPublicPluginSetup } from '../../triggers_actions_ui
 import { TypeRegistry } from '../../triggers_actions_ui/public/application/type_registry';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 import { ActionTypeModel, AlertTypeModel } from '../../triggers_actions_ui/public/types';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
 
 interface BreadcrumbItem {
   ['data-test-subj']?: string;
@@ -59,13 +60,14 @@ export interface IShims {
   ) => Promise<any>;
   isCloud: boolean;
   triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
+  usageCollection: UsageCollectionSetup;
 }
 
 export class Legacy {
   private static _shims: IShims;
 
   public static init(
-    { core, data, isCloud, triggersActionsUi }: MonitoringStartPluginDependencies,
+    { core, data, isCloud, triggersActionsUi, usageCollection }: MonitoringStartPluginDependencies,
     ngInjector: angular.auto.IInjectorService
   ) {
     this._shims = {
@@ -119,6 +121,7 @@ export class Legacy {
         }),
       isCloud,
       triggersActionsUi,
+      usageCollection,
     };
   }
 
diff --git a/x-pack/plugins/monitoring/public/lib/setup_mode.tsx b/x-pack/plugins/monitoring/public/lib/setup_mode.tsx
index 3425e0ee2a8188..3e555c843a0bb2 100644
--- a/x-pack/plugins/monitoring/public/lib/setup_mode.tsx
+++ b/x-pack/plugins/monitoring/public/lib/setup_mode.tsx
@@ -8,6 +8,7 @@ import React from 'react';
 import { render } from 'react-dom';
 import { get, includes } from 'lodash';
 import { i18n } from '@kbn/i18n';
+import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public';
 import { Legacy } from '../legacy_shims';
 import { ajaxErrorHandlersProvider } from './ajax_error_handler';
 import { SetupModeEnterButton } from '../components/setup_mode/enter_button';
@@ -179,8 +180,17 @@ export const setSetupModeMenuItem = () => {
   const globalState = angularState.injector.get('globalState');
   const enabled = !globalState.inSetupMode;
 
+  const services = {
+    usageCollection: Legacy.shims.usageCollection,
+  };
+  const I18nContext = Legacy.shims.I18nContext;
+
   render(
-    <SetupModeEnterButton enabled={enabled} toggleSetupMode={toggleSetupMode} />,
+    <KibanaContextProvider services={services}>
+      <I18nContext>
+        <SetupModeEnterButton enabled={enabled} toggleSetupMode={toggleSetupMode} />
+      </I18nContext>
+    </KibanaContextProvider>,
     document.getElementById('setupModeNav')
   );
 };
diff --git a/x-pack/plugins/monitoring/public/plugin.ts b/x-pack/plugins/monitoring/public/plugin.ts
index 05aa75f5862410..087e7acc4c7033 100644
--- a/x-pack/plugins/monitoring/public/plugin.ts
+++ b/x-pack/plugins/monitoring/public/plugin.ts
@@ -13,6 +13,7 @@ import {
   Plugin,
   PluginInitializerContext,
 } from 'kibana/public';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
 import {
   FeatureCatalogueCategory,
   HomePublicPluginSetup,
@@ -28,6 +29,7 @@ interface MonitoringSetupPluginDependencies {
   home?: HomePublicPluginSetup;
   cloud?: { isCloudEnabled: boolean };
   triggers_actions_ui: TriggersAndActionsUIPublicPluginSetup;
+  usageCollection: UsageCollectionSetup;
 }
 
 export class MonitoringPlugin
@@ -93,6 +95,7 @@ export class MonitoringPlugin
           pluginInitializerContext: this.initializerContext,
           externalConfig: this.getExternalConfig(),
           triggersActionsUi: plugins.triggers_actions_ui,
+          usageCollection: plugins.usageCollection,
         };
 
         pluginsStart.kibanaLegacy.loadFontAwesome();
diff --git a/x-pack/plugins/monitoring/public/types.ts b/x-pack/plugins/monitoring/public/types.ts
index f911af2db8c58c..238af7276d5860 100644
--- a/x-pack/plugins/monitoring/public/types.ts
+++ b/x-pack/plugins/monitoring/public/types.ts
@@ -9,6 +9,7 @@ import { NavigationPublicPluginStart as NavigationStart } from '../../../../src/
 import { DataPublicPluginStart } from '../../../../src/plugins/data/public';
 import { TriggersAndActionsUIPublicPluginSetup } from '../../triggers_actions_ui/public';
 import { KibanaLegacyStart } from '../../../../src/plugins/kibana_legacy/public';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
 
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 export { MonitoringConfig } from '../server';
@@ -23,4 +24,5 @@ export interface MonitoringStartPluginDependencies {
   pluginInitializerContext: PluginInitializerContext;
   externalConfig: Array<Array<string | number> | Array<string | boolean>>;
   triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
+  usageCollection: UsageCollectionSetup;
 }
diff --git a/x-pack/plugins/monitoring/public/views/apm/instance/index.js b/x-pack/plugins/monitoring/public/views/apm/instance/index.js
index 752c46b18bfb45..752128782194e9 100644
--- a/x-pack/plugins/monitoring/public/views/apm/instance/index.js
+++ b/x-pack/plugins/monitoring/public/views/apm/instance/index.js
@@ -44,6 +44,7 @@ uiRoutes.when('/apm/instances/:uuid', {
             apm: 'APM server',
           },
         }),
+        telemetryPageViewTitle: 'apm_server_instance',
         api: `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/apm/${$route.current.params.uuid}`,
         defaultData: {},
         reactNodeId: 'apmInstanceReact',
@@ -63,21 +64,16 @@ uiRoutes.when('/apm/instances/:uuid', {
             })
           );
           title($scope.cluster, `APM server - ${get(data, 'apmSummary.name')}`);
-          this.renderReact(data);
+          this.renderReact(
+            <ApmServerInstance
+              summary={data.apmSummary || {}}
+              metrics={data.metrics || {}}
+              onBrush={this.onBrush}
+              zoomInfo={this.zoomInfo}
+            />
+          );
         }
       );
     }
-
-    renderReact(data) {
-      const component = (
-        <ApmServerInstance
-          summary={data.apmSummary || {}}
-          metrics={data.metrics || {}}
-          onBrush={this.onBrush}
-          zoomInfo={this.zoomInfo}
-        />
-      );
-      super.renderReact(component);
-    }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/apm/instances/index.js b/x-pack/plugins/monitoring/public/views/apm/instances/index.js
index 764e13ccfea8d3..1f5b089ea748e4 100644
--- a/x-pack/plugins/monitoring/public/views/apm/instances/index.js
+++ b/x-pack/plugins/monitoring/public/views/apm/instances/index.js
@@ -55,37 +55,33 @@ uiRoutes.when('/apm/instances', {
       $scope.$watch(
         () => this.data,
         (data) => {
-          this.renderReact(data);
-        }
-      );
-    }
+          const { pagination, sorting, onTableChange } = this;
 
-    renderReact(data) {
-      const { pagination, sorting, onTableChange } = this;
-
-      const component = (
-        <SetupModeRenderer
-          scope={this.scope}
-          injector={this.injector}
-          productName={APM_SYSTEM_ID}
-          render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
-            <Fragment>
-              {flyoutComponent}
-              <ApmServerInstances
-                setupMode={setupMode}
-                apms={{
-                  pagination,
-                  sorting,
-                  onTableChange,
-                  data,
-                }}
-              />
-              {bottomBarComponent}
-            </Fragment>
-          )}
-        />
+          const component = (
+            <SetupModeRenderer
+              scope={this.scope}
+              injector={this.injector}
+              productName={APM_SYSTEM_ID}
+              render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
+                <Fragment>
+                  {flyoutComponent}
+                  <ApmServerInstances
+                    setupMode={setupMode}
+                    apms={{
+                      pagination,
+                      sorting,
+                      onTableChange,
+                      data,
+                    }}
+                  />
+                  {bottomBarComponent}
+                </Fragment>
+              )}
+            />
+          );
+          this.renderReact(component);
+        }
       );
-      super.renderReact(component);
     }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/apm/overview/index.js b/x-pack/plugins/monitoring/public/views/apm/overview/index.js
index 670acaeacce033..544fae39ee79d3 100644
--- a/x-pack/plugins/monitoring/public/views/apm/overview/index.js
+++ b/x-pack/plugins/monitoring/public/views/apm/overview/index.js
@@ -47,14 +47,11 @@ uiRoutes.when('/apm', {
       $scope.$watch(
         () => this.data,
         (data) => {
-          this.renderReact(data);
+          this.renderReact(
+            <ApmOverview {...data} onBrush={this.onBrush} zoomInfo={this.zoomInfo} />
+          );
         }
       );
     }
-
-    renderReact(data) {
-      const component = <ApmOverview {...data} onBrush={this.onBrush} zoomInfo={this.zoomInfo} />;
-      super.renderReact(component);
-    }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/base_controller.js b/x-pack/plugins/monitoring/public/views/base_controller.js
index 9c5aef950fc2bf..0eb40c8dd5963d 100644
--- a/x-pack/plugins/monitoring/public/views/base_controller.js
+++ b/x-pack/plugins/monitoring/public/views/base_controller.js
@@ -13,6 +13,7 @@ import { Legacy } from '../legacy_shims';
 import { PromiseWithCancel } from '../../common/cancel_promise';
 import { SetupModeFeature } from '../../common/enums';
 import { updateSetupModeData, isSetupModeFeatureEnabled } from '../lib/setup_mode';
+import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public';
 
 /**
  * Given a timezone, this function will calculate the offset in milliseconds
@@ -89,6 +90,7 @@ export class MonitoringViewBaseController {
     options = {},
     alerts = { shouldFetch: false, options: {} },
     fetchDataImmediately = true,
+    telemetryPageViewTitle = '',
   }) {
     const titleService = $injector.get('title');
     const $executor = $injector.get('$executor');
@@ -102,6 +104,7 @@ export class MonitoringViewBaseController {
     $scope.pageData = this.data = { ...defaultData };
     this._isDataInitialized = false;
     this.reactNodeId = reactNodeId;
+    this.telemetryPageViewTitle = telemetryPageViewTitle || title;
 
     let deferTimer;
     let zoomInLevel = 0;
@@ -207,6 +210,8 @@ export class MonitoringViewBaseController {
         deferTimer = setTimeout(() => addPopstateHandler(), 10);
       };
 
+      // Render loading state
+      this.renderReact(null, true);
       fetchDataImmediately && this.updateData();
     });
 
@@ -228,15 +233,26 @@ export class MonitoringViewBaseController {
     this.setTitle = (title) => titleService($scope.cluster, title);
   }
 
-  renderReact(component) {
+  renderReact(component, trackPageView = false) {
     const renderElement = document.getElementById(this.reactNodeId);
     if (!renderElement) {
       console.warn(`"#${this.reactNodeId}" element has not been added to the DOM yet`);
       return;
     }
+    const services = {
+      usageCollection: Legacy.shims.usageCollection,
+    };
     const I18nContext = Legacy.shims.I18nContext;
     const wrappedComponent = (
-      <I18nContext>{!this._isDataInitialized ? <PageLoading /> : component}</I18nContext>
+      <KibanaContextProvider services={services}>
+        <I18nContext>
+          {!this._isDataInitialized ? (
+            <PageLoading pageViewTitle={trackPageView ? this.telemetryPageViewTitle : null} />
+          ) : (
+            component
+          )}
+        </I18nContext>
+      </KibanaContextProvider>
     );
     render(wrappedComponent, renderElement);
   }
diff --git a/x-pack/plugins/monitoring/public/views/beats/beat/index.js b/x-pack/plugins/monitoring/public/views/beats/beat/index.js
index 70a9f33b4f03df..6cffae2479128d 100644
--- a/x-pack/plugins/monitoring/public/views/beats/beat/index.js
+++ b/x-pack/plugins/monitoring/public/views/beats/beat/index.js
@@ -47,6 +47,7 @@ uiRoutes.when('/beats/beat/:beatUuid', {
             beatName: pageData.summary.name,
           },
         }),
+        telemetryPageViewTitle: 'beats_instance',
         getPageData,
         $scope,
         $injector,
diff --git a/x-pack/plugins/monitoring/public/views/beats/listing/index.js b/x-pack/plugins/monitoring/public/views/beats/listing/index.js
index 004f89adf0467f..a1b2231901c54c 100644
--- a/x-pack/plugins/monitoring/public/views/beats/listing/index.js
+++ b/x-pack/plugins/monitoring/public/views/beats/listing/index.js
@@ -40,6 +40,7 @@ uiRoutes.when('/beats/beats', {
         pageTitle: i18n.translate('xpack.monitoring.beats.listing.pageTitle', {
           defaultMessage: 'Beats listing',
         }),
+        telemetryPageViewTitle: 'beats_listing',
         storageKey: 'beats.beats',
         getPageData,
         reactNodeId: 'monitoringBeatsInstancesApp',
@@ -51,9 +52,6 @@ uiRoutes.when('/beats/beats', {
       this.scope = $scope;
       this.injector = $injector;
 
-      //Bypassing super.updateData, since this controller loads its own data
-      this._isDataInitialized = true;
-
       $scope.$watch(
         () => this.data,
         () => this.renderComponent()
diff --git a/x-pack/plugins/monitoring/public/views/cluster/listing/index.js b/x-pack/plugins/monitoring/public/views/cluster/listing/index.js
index b1e850ef3a905f..dd984f559d4698 100644
--- a/x-pack/plugins/monitoring/public/views/cluster/listing/index.js
+++ b/x-pack/plugins/monitoring/public/views/cluster/listing/index.js
@@ -52,6 +52,7 @@ uiRoutes
           $scope,
           $injector,
           reactNodeId: 'monitoringClusterListingApp',
+          telemetryPageViewTitle: 'cluster_listing',
         });
 
         const $route = $injector.get('$route');
diff --git a/x-pack/plugins/monitoring/public/views/cluster/overview/index.js b/x-pack/plugins/monitoring/public/views/cluster/overview/index.js
index 91537d5c77ba42..6f27a12223b26f 100644
--- a/x-pack/plugins/monitoring/public/views/cluster/overview/index.js
+++ b/x-pack/plugins/monitoring/public/views/cluster/overview/index.js
@@ -54,6 +54,7 @@ uiRoutes.when('/overview', {
         alerts: {
           shouldFetch: true,
         },
+        telemetryPageViewTitle: 'cluster_overview',
       });
 
       this.init = () => this.renderReact(null);
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js
index 32b3546510f91a..65693407857363 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js
@@ -42,13 +42,12 @@ uiRoutes.when('/elasticsearch/ccr', {
       $scope.$watch(
         () => this.data,
         (data) => {
-          this.renderReact(data);
+          if (!data) {
+            return;
+          }
+          this.renderReact(<Ccr data={data.data} />);
         }
       );
-
-      this.renderReact = ({ data }) => {
-        super.renderReact(<Ccr data={data} />);
-      };
     }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js
index e60a29ea8a5edb..33a2d27f398565 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js
@@ -48,6 +48,10 @@ uiRoutes.when('/elasticsearch/ccr/:index/shard/:shardId', {
       $scope.$watch(
         () => this.data,
         (data) => {
+          if (!data) {
+            return;
+          }
+
           this.setPageTitle(
             i18n.translate('xpack.monitoring.elasticsearch.ccr.shard.pageTitle', {
               defaultMessage: 'Elasticsearch Ccr Shard - Index: {followerIndex} Shard: {shardId}',
@@ -57,13 +61,10 @@ uiRoutes.when('/elasticsearch/ccr/:index/shard/:shardId', {
               },
             })
           );
-          this.renderReact(data);
+
+          this.renderReact(<CcrShard {...data} />);
         }
       );
-
-      this.renderReact = (props) => {
-        super.renderReact(<CcrShard {...props} />);
-      };
     }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/index/advanced/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/index/advanced/index.js
index f4b0f0789bae13..cfc36e360709d8 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/index/advanced/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/index/advanced/index.js
@@ -64,6 +64,7 @@ uiRoutes.when('/elasticsearch/indices/:index/advanced', {
             indexName,
           },
         }),
+        telemetryPageViewTitle: 'elasticsearch_index_advanced',
         defaultData: {},
         getPageData,
         reactNodeId: 'monitoringElasticsearchAdvancedIndexApp',
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/index/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/index/index.js
index a7ca4b5b87ab08..76628a0a02e42d 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/index/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/index/index.js
@@ -66,6 +66,7 @@ uiRoutes.when('/elasticsearch/indices/:index', {
             indexName,
           },
         }),
+        telemetryPageViewTitle: 'elasticsearch_index',
         pageTitle: i18n.translate('xpack.monitoring.elasticsearch.indices.overview.pageTitle', {
           defaultMessage: 'Index: {indexName}',
           values: {
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/indices/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/indices/index.js
index 911f7146d7282e..490bd02db42b7b 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/indices/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/indices/index.js
@@ -67,23 +67,24 @@ uiRoutes.when('/elasticsearch/indices', {
       $scope.$watch(
         () => this.data,
         (data) => {
-          this.renderReact(data);
+          if (!data) {
+            return;
+          }
+
+          const { clusterStatus, indices } = data;
+          this.renderReact(
+            <ElasticsearchIndices
+              clusterStatus={clusterStatus}
+              indices={indices}
+              showSystemIndices={showSystemIndices}
+              toggleShowSystemIndices={toggleShowSystemIndices}
+              sorting={this.sorting}
+              pagination={this.pagination}
+              onTableChange={this.onTableChange}
+            />
+          );
         }
       );
-
-      this.renderReact = ({ clusterStatus, indices }) => {
-        super.renderReact(
-          <ElasticsearchIndices
-            clusterStatus={clusterStatus}
-            indices={indices}
-            showSystemIndices={showSystemIndices}
-            toggleShowSystemIndices={toggleShowSystemIndices}
-            sorting={this.sorting}
-            pagination={this.pagination}
-            onTableChange={this.onTableChange}
-          />
-        );
-      };
     }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js
index b5d498950ef075..5c4b4d28b93cb9 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js
@@ -61,6 +61,7 @@ uiRoutes.when('/elasticsearch/nodes/:node/advanced', {
         defaultData: {},
         getPageData,
         reactNodeId: 'monitoringElasticsearchAdvancedNodeApp',
+        telemetryPageViewTitle: 'elasticsearch_node_advanced',
         $scope,
         $injector,
         alerts: {
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js
index faff1e09aff034..b4b3c7ca553031 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js
@@ -36,6 +36,13 @@ uiRoutes.when('/elasticsearch/nodes/:node', {
       const nodeName = $route.current.params.node;
 
       super({
+        title: i18n.translate('xpack.monitoring.elasticsearch.node.overview.routeTitle', {
+          defaultMessage: 'Elasticsearch - Nodes - {nodeName} - Overview',
+          values: {
+            nodeName,
+          },
+        }),
+        telemetryPageViewTitle: 'elasticsearch_node',
         defaultData: {},
         getPageData,
         reactNodeId: 'monitoringElasticsearchNodeApp',
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js
index 2a1172105b0736..33584f802a56ed 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js
@@ -95,38 +95,41 @@ uiRoutes.when('/elasticsearch/nodes', {
 
       $scope.$watch(
         () => this.data,
-        () => this.renderReact(this.data || {})
-      );
+        (data) => {
+          if (!data) {
+            return;
+          }
 
-      this.renderReact = ({ clusterStatus, nodes, totalNodeCount }) => {
-        const pagination = {
-          ...this.pagination,
-          totalItemCount: totalNodeCount,
-        };
+          const { clusterStatus, nodes, totalNodeCount } = data;
+          const pagination = {
+            ...this.pagination,
+            totalItemCount: totalNodeCount,
+          };
 
-        super.renderReact(
-          <SetupModeRenderer
-            scope={$scope}
-            injector={$injector}
-            productName={ELASTICSEARCH_SYSTEM_ID}
-            render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
-              <Fragment>
-                {flyoutComponent}
-                <ElasticsearchNodes
-                  clusterStatus={clusterStatus}
-                  clusterUuid={globalState.cluster_uuid}
-                  setupMode={setupMode}
-                  nodes={nodes}
-                  alerts={this.alerts}
-                  showCgroupMetricsElasticsearch={showCgroupMetricsElasticsearch}
-                  {...this.getPaginationTableProps(pagination)}
-                />
-                {bottomBarComponent}
-              </Fragment>
-            )}
-          />
-        );
-      };
+          this.renderReact(
+            <SetupModeRenderer
+              scope={$scope}
+              injector={$injector}
+              productName={ELASTICSEARCH_SYSTEM_ID}
+              render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
+                <Fragment>
+                  {flyoutComponent}
+                  <ElasticsearchNodes
+                    clusterStatus={clusterStatus}
+                    clusterUuid={globalState.cluster_uuid}
+                    setupMode={setupMode}
+                    nodes={nodes}
+                    alerts={this.alerts}
+                    showCgroupMetricsElasticsearch={showCgroupMetricsElasticsearch}
+                    {...this.getPaginationTableProps(pagination)}
+                  />
+                  {bottomBarComponent}
+                </Fragment>
+              )}
+            />
+          );
+        }
+      );
     }
   },
 });
diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/overview/controller.js b/x-pack/plugins/monitoring/public/views/elasticsearch/overview/controller.js
index 5dca8a2dbd9071..f383b36bb3524c 100644
--- a/x-pack/plugins/monitoring/public/views/elasticsearch/overview/controller.js
+++ b/x-pack/plugins/monitoring/public/views/elasticsearch/overview/controller.js
@@ -78,7 +78,7 @@ export class ElasticsearchOverviewController extends MonitoringViewBaseControlle
 
   renderReact(data, cluster) {
     // All data needs to originate in this view, and get passed as a prop to the components, for statelessness
-    const { clusterStatus, metrics, shardActivity, logs } = data;
+    const { clusterStatus, metrics, shardActivity, logs } = data || {};
     const shardActivityData = shardActivity && this.filterShardActivityData(shardActivity); // no filter on data = null
     const component = (
       <ElasticsearchOverview
diff --git a/x-pack/plugins/monitoring/public/views/kibana/instance/index.js b/x-pack/plugins/monitoring/public/views/kibana/instance/index.js
index 1dc4dac4c26171..20a1a517194156 100644
--- a/x-pack/plugins/monitoring/public/views/kibana/instance/index.js
+++ b/x-pack/plugins/monitoring/public/views/kibana/instance/index.js
@@ -67,6 +67,7 @@ uiRoutes.when('/kibana/instances/:uuid', {
     constructor($injector, $scope) {
       super({
         title: `Kibana - ${get($scope.pageData, 'kibanaSummary.name')}`,
+        telemetryPageViewTitle: 'kibana_instance',
         defaultData: {},
         getPageData,
         reactNodeId: 'monitoringKibanaInstanceApp',
@@ -86,7 +87,6 @@ uiRoutes.when('/kibana/instances/:uuid', {
           if (!data || !data.metrics) {
             return;
           }
-
           this.setTitle(`Kibana - ${get(data, 'kibanaSummary.name')}`);
           this.setPageTitle(
             i18n.translate('xpack.monitoring.kibana.instance.pageTitle', {
diff --git a/x-pack/plugins/monitoring/public/views/logstash/node/advanced/index.js b/x-pack/plugins/monitoring/public/views/logstash/node/advanced/index.js
index d87b3468aaec03..0fc3cc47502cd6 100644
--- a/x-pack/plugins/monitoring/public/views/logstash/node/advanced/index.js
+++ b/x-pack/plugins/monitoring/public/views/logstash/node/advanced/index.js
@@ -69,6 +69,7 @@ uiRoutes.when('/logstash/node/:uuid/advanced', {
         reactNodeId: 'monitoringLogstashNodeAdvancedApp',
         $scope,
         $injector,
+        telemetryPageViewTitle: 'logstash_node_advanced',
       });
 
       $scope.$watch(
diff --git a/x-pack/plugins/monitoring/public/views/logstash/node/index.js b/x-pack/plugins/monitoring/public/views/logstash/node/index.js
index 08f28c64c43ed3..e2dee77133c72b 100644
--- a/x-pack/plugins/monitoring/public/views/logstash/node/index.js
+++ b/x-pack/plugins/monitoring/public/views/logstash/node/index.js
@@ -76,6 +76,7 @@ uiRoutes.when('/logstash/node/:uuid', {
             alertTypeIds: [ALERT_LOGSTASH_VERSION_MISMATCH],
           },
         },
+        telemetryPageViewTitle: 'logstash_node',
       });
 
       $scope.$watch(
diff --git a/x-pack/plugins/monitoring/public/views/logstash/node/pipelines/index.js b/x-pack/plugins/monitoring/public/views/logstash/node/pipelines/index.js
index 5324dca32a0f19..685bb5f5a64c7f 100644
--- a/x-pack/plugins/monitoring/public/views/logstash/node/pipelines/index.js
+++ b/x-pack/plugins/monitoring/public/views/logstash/node/pipelines/index.js
@@ -81,6 +81,7 @@ uiRoutes.when('/logstash/node/:uuid/pipelines', {
         $scope,
         $injector,
         fetchDataImmediately: false, // We want to apply pagination before sending the first request
+        telemetryPageViewTitle: 'logstash_node_pipelines',
       });
 
       $scope.$watch(
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.test.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.test.ts
new file mode 100644
index 00000000000000..17caf80eef22bd
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.test.ts
@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getMonitoringUsageCollector } from './get_usage_collector';
+import { fetchClusters } from '../../lib/alerts/fetch_clusters';
+
+jest.mock('../../lib/alerts/fetch_clusters', () => ({
+  fetchClusters: jest.fn().mockImplementation(() => {
+    return [
+      {
+        clusterUuid: '1abc',
+        clusterName: 'unitTesting',
+      },
+    ];
+  }),
+}));
+
+jest.mock('./lib/get_stack_products_usage', () => ({
+  getStackProductsUsage: jest.fn().mockImplementation(() => {
+    return {
+      elasticsearch: {
+        count: 5,
+        enabled: true,
+        metricbeatUsed: true,
+      },
+      kibana: {
+        count: 2,
+        enabled: true,
+        metricbeatUsed: false,
+      },
+      logstash: {
+        count: 0,
+        enabled: false,
+        metricbeatUsed: false,
+      },
+      beats: {
+        count: 1,
+        enabled: true,
+        metricbeatUsed: false,
+      },
+      apm: {
+        count: 1,
+        enabled: true,
+        metricbeatUsed: true,
+      },
+    };
+  }),
+}));
+
+jest.mock('./lib/fetch_license_type', () => ({
+  fetchLicenseType: jest.fn().mockImplementation(() => {
+    return 'trial';
+  }),
+}));
+
+describe('getMonitoringUsageCollector', () => {
+  const callCluster = jest.fn();
+  const config: any = {
+    ui: {
+      ccs: {
+        enabled: true,
+      },
+    },
+  };
+
+  it('should be configured correctly', async () => {
+    const usageCollection: any = {
+      makeUsageCollector: jest.fn(),
+    };
+    await getMonitoringUsageCollector(usageCollection, config, callCluster);
+
+    const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
+
+    const args = mock.calls[0];
+    expect(args[0].type).toBe('monitoring');
+    expect(typeof args[0].isReady).toBe('function');
+    expect(args[0].schema).toStrictEqual({
+      hasMonitoringData: { type: 'boolean' },
+      clusters: {
+        license: { type: 'keyword' },
+        clusterUuid: { type: 'keyword' },
+        metricbeatUsed: { type: 'boolean' },
+        elasticsearch: {
+          enabled: { type: 'boolean' },
+          count: { type: 'long' },
+          metricbeatUsed: { type: 'boolean' },
+        },
+        kibana: {
+          enabled: { type: 'boolean' },
+          count: { type: 'long' },
+          metricbeatUsed: { type: 'boolean' },
+        },
+        logstash: {
+          enabled: { type: 'boolean' },
+          count: { type: 'long' },
+          metricbeatUsed: { type: 'boolean' },
+        },
+        beats: {
+          enabled: { type: 'boolean' },
+          count: { type: 'long' },
+          metricbeatUsed: { type: 'boolean' },
+        },
+        apm: {
+          enabled: { type: 'boolean' },
+          count: { type: 'long' },
+          metricbeatUsed: { type: 'boolean' },
+        },
+      },
+    });
+  });
+
+  it('should fetch usage data', async () => {
+    const usageCollection: any = {
+      makeUsageCollector: jest.fn(),
+    };
+
+    await getMonitoringUsageCollector(usageCollection, config, callCluster);
+    const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
+    const args = mock.calls[0];
+
+    const result = await args[0].fetch();
+    expect(result).toStrictEqual({
+      hasMonitoringData: true,
+      clusters: [
+        {
+          clusterUuid: '1abc',
+          license: 'trial',
+          elasticsearch: { count: 5, enabled: true, metricbeatUsed: true },
+          kibana: { count: 2, enabled: true, metricbeatUsed: false },
+          logstash: { count: 0, enabled: false, metricbeatUsed: false },
+          beats: { count: 1, enabled: true, metricbeatUsed: false },
+          apm: { count: 1, enabled: true, metricbeatUsed: true },
+          metricbeatUsed: true,
+        },
+      ],
+    });
+  });
+
+  it('should handle no monitoring data', async () => {
+    const usageCollection: any = {
+      makeUsageCollector: jest.fn(),
+    };
+
+    await getMonitoringUsageCollector(usageCollection, config, callCluster);
+    const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
+    const args = mock.calls[0];
+
+    (fetchClusters as jest.Mock).mockImplementation(() => {
+      return [];
+    });
+
+    const result = await args[0].fetch();
+    expect(result).toStrictEqual({
+      hasMonitoringData: false,
+      clusters: [],
+    });
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts
new file mode 100644
index 00000000000000..b743a5f8e0b4fa
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { LegacyAPICaller } from 'src/core/server';
+import { MonitoringConfig } from '../../config';
+import { fetchAvailableCcs } from '../../lib/alerts/fetch_available_ccs';
+import { getStackProductsUsage } from './lib/get_stack_products_usage';
+import { fetchLicenseType } from './lib/fetch_license_type';
+import { MonitoringUsage, StackProductUsage, MonitoringClusterStackProductUsage } from './types';
+import { INDEX_PATTERN_ELASTICSEARCH } from '../../../common/constants';
+import { getCcsIndexPattern } from '../../lib/alerts/get_ccs_index_pattern';
+import { fetchClusters } from '../../lib/alerts/fetch_clusters';
+
+export function getMonitoringUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  config: MonitoringConfig,
+  callCluster: LegacyAPICaller
+) {
+  return usageCollection.makeUsageCollector<MonitoringUsage>({
+    type: 'monitoring',
+    isReady: () => true,
+    schema: {
+      hasMonitoringData: {
+        type: 'boolean',
+      },
+      clusters: {
+        license: {
+          type: 'keyword',
+        },
+        clusterUuid: {
+          type: 'keyword',
+        },
+        metricbeatUsed: {
+          type: 'boolean',
+        },
+        elasticsearch: {
+          enabled: {
+            type: 'boolean',
+          },
+          count: {
+            type: 'long',
+          },
+          metricbeatUsed: {
+            type: 'boolean',
+          },
+        },
+        kibana: {
+          enabled: {
+            type: 'boolean',
+          },
+          count: {
+            type: 'long',
+          },
+          metricbeatUsed: {
+            type: 'boolean',
+          },
+        },
+        logstash: {
+          enabled: {
+            type: 'boolean',
+          },
+          count: {
+            type: 'long',
+          },
+          metricbeatUsed: {
+            type: 'boolean',
+          },
+        },
+        beats: {
+          enabled: {
+            type: 'boolean',
+          },
+          count: {
+            type: 'long',
+          },
+          metricbeatUsed: {
+            type: 'boolean',
+          },
+        },
+        apm: {
+          enabled: {
+            type: 'boolean',
+          },
+          count: {
+            type: 'long',
+          },
+          metricbeatUsed: {
+            type: 'boolean',
+          },
+        },
+      },
+    },
+    fetch: async () => {
+      const usageClusters: MonitoringClusterStackProductUsage[] = [];
+      const availableCcs = config.ui.ccs.enabled ? await fetchAvailableCcs(callCluster) : [];
+      const elasticsearchIndex = getCcsIndexPattern(INDEX_PATTERN_ELASTICSEARCH, availableCcs);
+      const clusters = await fetchClusters(callCluster, elasticsearchIndex);
+      for (const cluster of clusters) {
+        const license = await fetchLicenseType(callCluster, availableCcs, cluster.clusterUuid);
+        const stackProducts = await getStackProductsUsage(
+          config,
+          callCluster,
+          availableCcs,
+          cluster.clusterUuid
+        );
+        usageClusters.push({
+          clusterUuid: cluster.clusterUuid,
+          license,
+          metricbeatUsed: Object.values(stackProducts).some(
+            (_usage: StackProductUsage) => _usage.metricbeatUsed
+          ),
+          ...stackProducts,
+        });
+      }
+
+      const usage = {
+        hasMonitoringData: usageClusters.length > 0,
+        clusters: usageClusters,
+      };
+
+      return usage;
+    },
+  });
+}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts
index aa4853ab226f49..47ad78b29962c6 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts
@@ -4,15 +4,21 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { LegacyAPICaller } from 'src/core/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { getSettingsCollector } from './get_settings_collector';
+import { getMonitoringUsageCollector } from './get_usage_collector';
 import { MonitoringConfig } from '../../config';
 
 export { KibanaSettingsCollector } from './get_settings_collector';
 
 export function registerCollectors(
   usageCollection: UsageCollectionSetup,
-  config: MonitoringConfig
+  config: MonitoringConfig,
+  callCluster: LegacyAPICaller
 ) {
   usageCollection.registerCollector(getSettingsCollector(usageCollection, config));
+  usageCollection.registerCollector(
+    getMonitoringUsageCollector(usageCollection, config, callCluster)
+  );
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.test.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.test.ts
new file mode 100644
index 00000000000000..85fc0eb8dc6b2f
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.test.ts
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { fetchESUsage } from './fetch_es_usage';
+
+describe('fetchESUsage', () => {
+  const clusterUuid = '1abcde2';
+  const index = '.monitoring-es-*';
+  const callCluster = jest.fn().mockImplementation(() => ({
+    hits: {
+      hits: [
+        {
+          _source: {
+            cluster_stats: {
+              nodes: {
+                count: {
+                  total: 10,
+                },
+              },
+            },
+          },
+        },
+      ],
+    },
+    aggregations: {
+      indices: {
+        buckets: [
+          {
+            key: '.monitoring-es-2',
+          },
+        ],
+      },
+    },
+  }));
+  const config: any = {};
+
+  it('should return usage data for Elasticsearch', async () => {
+    const result = await fetchESUsage(config, callCluster, clusterUuid, index);
+    expect(result).toStrictEqual({
+      count: 10,
+      enabled: true,
+      metricbeatUsed: false,
+    });
+  });
+
+  it('should handle some indices coming from Metricbeat', async () => {
+    const customCallCluster = jest.fn().mockImplementation(() => ({
+      hits: {
+        hits: [
+          {
+            _source: {
+              cluster_stats: {
+                nodes: {
+                  count: {
+                    total: 10,
+                  },
+                },
+              },
+            },
+          },
+        ],
+      },
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-mb-2',
+            },
+          ],
+        },
+      },
+    }));
+    const result = await fetchESUsage(config, customCallCluster, clusterUuid, index);
+    expect(result).toStrictEqual({
+      count: 10,
+      enabled: true,
+      metricbeatUsed: true,
+    });
+  });
+
+  it('should handle no monitoring data', async () => {
+    const customCallCluster = jest.fn().mockImplementation(() => ({
+      hits: {
+        hits: [],
+      },
+    }));
+    const result = await fetchESUsage(config, customCallCluster, clusterUuid, index);
+    expect(result).toStrictEqual({
+      count: 0,
+      enabled: false,
+      metricbeatUsed: false,
+    });
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.ts
new file mode 100644
index 00000000000000..de0a1b8f99d968
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_es_usage.ts
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from 'src/core/server';
+import { get } from 'lodash';
+import { MonitoringConfig } from '../../../config';
+import { StackProductUsage } from '../types';
+
+interface ESResponse {
+  hits: {
+    hits: ESResponseHits[];
+  };
+  aggregations: {
+    indices: {
+      buckets: ESIndicesBucket;
+    };
+  };
+}
+
+interface ESIndicesBucket {
+  key: string;
+}
+
+interface ESResponseHits {
+  _source: ClusterStats;
+}
+
+interface ClusterStats {
+  cluster_stats: {
+    nodes: {
+      count: {
+        total: number;
+      };
+    };
+  };
+  version: string;
+}
+
+export async function fetchESUsage(
+  config: MonitoringConfig,
+  callCluster: LegacyAPICaller,
+  clusterUuid: string,
+  index: string
+): Promise<StackProductUsage> {
+  const params = {
+    index,
+    size: 1,
+    ignoreUnavailable: true,
+    filterPath: [
+      'hits.hits._source.cluster_stats.nodes.count.total',
+      'aggregations.indices.buckets',
+    ],
+    body: {
+      sort: [
+        {
+          timestamp: {
+            order: 'desc',
+          },
+        },
+      ],
+      query: {
+        bool: {
+          must: [
+            {
+              term: {
+                type: {
+                  value: 'cluster_stats',
+                },
+              },
+            },
+            {
+              term: {
+                cluster_uuid: {
+                  value: clusterUuid,
+                },
+              },
+            },
+            {
+              range: {
+                timestamp: {
+                  gte: 'now-1h',
+                },
+              },
+            },
+          ],
+        },
+      },
+      aggs: {
+        indices: {
+          terms: {
+            field: '_index',
+            size: 2,
+          },
+        },
+      },
+    },
+  };
+
+  const response = await callCluster('search', params);
+  const esResponse = response as ESResponse;
+  if (esResponse.hits.hits.length === 0) {
+    return {
+      count: 0,
+      enabled: false,
+      metricbeatUsed: false,
+    };
+  }
+
+  const hit = esResponse.hits.hits[0]._source;
+  const count = hit.cluster_stats.nodes.count.total;
+  const buckets = get(esResponse, 'aggregations.indices.buckets', []) as ESIndicesBucket[];
+  const metricbeatUsed = Boolean(buckets.find((indexBucket) => indexBucket.key.includes('-mb-')));
+
+  return {
+    count,
+    enabled: true,
+    metricbeatUsed,
+  };
+}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.test.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.test.ts
new file mode 100644
index 00000000000000..1026dc339e29e1
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.test.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { fetchLicenseType } from './fetch_license_type';
+
+describe('fetchLicenseType', () => {
+  const clusterUuid = '1abcde2';
+  const availableCcs: string[] = [];
+  const callCluster = jest.fn().mockImplementation(() => ({
+    hits: {
+      hits: [
+        {
+          _source: {
+            license: {
+              type: 'trial',
+            },
+          },
+        },
+      ],
+    },
+  }));
+
+  it('should get the license type', async () => {
+    const result = await fetchLicenseType(callCluster, availableCcs, clusterUuid);
+    expect(result).toStrictEqual('trial');
+  });
+
+  it('should handle no license data', async () => {
+    const customCallCluster = jest.fn().mockImplementation(() => ({
+      hits: {
+        hits: [],
+      },
+    }));
+    const result = await fetchLicenseType(customCallCluster, availableCcs, clusterUuid);
+    expect(result).toStrictEqual(null);
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.ts
new file mode 100644
index 00000000000000..f7b8b72637b1ff
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_license_type.ts
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get } from 'lodash';
+import { LegacyAPICaller } from 'src/core/server';
+import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../common/constants';
+import { getCcsIndexPattern } from '../../../lib/alerts/get_ccs_index_pattern';
+
+export async function fetchLicenseType(
+  callCluster: LegacyAPICaller,
+  availableCcs: string[],
+  clusterUuid: string
+) {
+  let index = INDEX_PATTERN_ELASTICSEARCH;
+  if (availableCcs) {
+    index = getCcsIndexPattern(index, availableCcs);
+  }
+  const params = {
+    index,
+    filterPath: ['hits.hits._source.license'],
+    body: {
+      size: 1,
+      sort: [
+        {
+          timestamp: {
+            order: 'desc',
+          },
+        },
+      ],
+      query: {
+        bool: {
+          must: [
+            {
+              term: {
+                cluster_uuid: {
+                  value: clusterUuid,
+                },
+              },
+            },
+            {
+              term: {
+                type: {
+                  value: 'cluster_stats',
+                },
+              },
+            },
+          ],
+        },
+      },
+    },
+  };
+  const response = await callCluster('search', params);
+  return get(response, 'hits.hits[0]._source.license.type', null);
+}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.test.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.test.ts
new file mode 100644
index 00000000000000..9377dee2f31f97
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.test.ts
@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { fetchStackProductUsage } from './fetch_stack_product_usage';
+
+describe('fetchStackProductUsage', () => {
+  const clusterUuid = '1abcde2';
+  const config: any = {
+    ui: {
+      max_bucket_size: 10000,
+    },
+  };
+
+  it('should use appropiate query parameters', async () => {
+    const callCluster = jest.fn();
+    await fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      '.monitoring-kibana-*',
+      'kibana_stats',
+      'kibana_stats.kibana.uuid',
+      [
+        {
+          term: {
+            type: {
+              value: 'foo',
+            },
+          },
+        },
+      ]
+    );
+    const params = callCluster.mock.calls[0][1];
+    expect(params.body.query.bool.must[0].term.type.value).toBe('kibana_stats');
+    expect(params.body.query.bool.must[1].term.cluster_uuid.value).toBe(clusterUuid);
+    expect(params.body.query.bool.must[2].range.timestamp.gte).toBe('now-1h');
+    expect(params.body.query.bool.must[3].term.type.value).toBe('foo');
+  });
+
+  it('should get the usage data', async () => {
+    const callCluster = jest.fn().mockImplementation(() => ({
+      aggregations: {
+        uuids: {
+          buckets: [
+            {
+              key: 'sadfsdf',
+              indices: {
+                buckets: [
+                  {
+                    key: '.monitoring-kibana-8',
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      },
+    }));
+
+    const result = await fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      '.monitoring-kibana-*',
+      'kibana_stats',
+      'kibana_stats.kibana.uuid'
+    );
+
+    expect(result).toStrictEqual({
+      count: 1,
+      enabled: true,
+      metricbeatUsed: false,
+    });
+  });
+
+  it('should handle both collection types', async () => {
+    const callCluster = jest.fn().mockImplementation(() => ({
+      aggregations: {
+        uuids: {
+          buckets: [
+            {
+              key: 'sadfsdf',
+              indices: {
+                buckets: [
+                  {
+                    key: '.monitoring-kibana-8',
+                  },
+                  {
+                    key: '.monitoring-kibana-mb-8',
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      },
+    }));
+
+    const result = await fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      '.monitoring-kibana-*',
+      'kibana_stats',
+      'kibana_stats.kibana.uuid'
+    );
+
+    expect(result).toStrictEqual({
+      count: 1,
+      enabled: true,
+      metricbeatUsed: true,
+    });
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.ts
new file mode 100644
index 00000000000000..df18b28d36c617
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.ts
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get } from 'lodash';
+import { LegacyAPICaller } from 'src/core/server';
+import { MonitoringConfig } from '../../../config';
+// @ts-ignore
+import { prefixIndexPattern } from '../../../lib/ccs_utils';
+import { StackProductUsage } from '../types';
+
+interface ESResponse {
+  aggregations?: {
+    uuids: {
+      buckets: UuidBucket[];
+    };
+  };
+}
+
+interface UuidBucket {
+  key: string;
+  indices: {
+    buckets: KeyBucket[];
+  };
+}
+
+interface KeyBucket {
+  key: string;
+}
+
+export async function fetchStackProductUsage(
+  config: MonitoringConfig,
+  callCluster: LegacyAPICaller,
+  clusterUuid: string,
+  index: string,
+  type: string,
+  uuidPath: string,
+  filters: any[] = []
+): Promise<StackProductUsage> {
+  const size = config.ui.max_bucket_size;
+  const params = {
+    index,
+    size: 0,
+    ignoreUnavailable: true,
+    filterPath: ['aggregations.uuids.buckets'],
+    body: {
+      query: {
+        bool: {
+          must: [
+            {
+              term: {
+                type: {
+                  value: type,
+                },
+              },
+            },
+            {
+              term: {
+                cluster_uuid: {
+                  value: clusterUuid,
+                },
+              },
+            },
+            {
+              range: {
+                timestamp: {
+                  gte: 'now-1h',
+                },
+              },
+            },
+            ...filters,
+          ],
+        },
+      },
+      aggs: {
+        uuids: {
+          terms: {
+            field: uuidPath,
+            size,
+          },
+          aggs: {
+            indices: {
+              terms: {
+                field: '_index',
+                size: 2,
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+
+  const response = (await callCluster('search', params)) as ESResponse;
+  const uuidBuckets = get(response, 'aggregations.uuids.buckets', []) as UuidBucket[];
+  const count = uuidBuckets.length;
+  const metricbeatUsed = Boolean(
+    uuidBuckets.find((uuidBucket) =>
+      (get(uuidBucket, 'indices.buckets', []) as KeyBucket[]).find((indexBucket) =>
+        indexBucket.key.includes('-mb-')
+      )
+    )
+  );
+  return {
+    count,
+    enabled: count > 0,
+    metricbeatUsed,
+  };
+}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.test.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.test.ts
new file mode 100644
index 00000000000000..ca8def84432ea4
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.test.ts
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getStackProductsUsage } from './get_stack_products_usage';
+
+describe('getStackProductsUsage', () => {
+  const config: any = {
+    ui: {
+      max_bucket_size: 10000,
+    },
+  };
+  const clusterUuid = '1abcde2';
+  const availableCcs: string[] = [];
+  const callCluster = jest.fn().mockImplementation(() => ({
+    hits: {
+      hits: [],
+    },
+  }));
+
+  it('should get all stack products', async () => {
+    const result = await getStackProductsUsage(config, callCluster, availableCcs, clusterUuid);
+    expect(result.elasticsearch).toBeDefined();
+    expect(result.kibana).toBeDefined();
+    expect(result.logstash).toBeDefined();
+    expect(result.beats).toBeDefined();
+    expect(result.apm).toBeDefined();
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.ts
new file mode 100644
index 00000000000000..ffa15168d5c8ae
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.ts
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from 'src/core/server';
+import { MonitoringClusterStackProductUsage } from '../types';
+import { fetchESUsage } from './fetch_es_usage';
+import { MonitoringConfig } from '../../../config';
+// @ts-ignore
+import { getIndexPatterns } from '../../../lib/cluster/get_index_patterns';
+// @ts-ignore
+import { prefixIndexPattern } from '../../../lib/ccs_utils';
+import {
+  INDEX_PATTERN_ELASTICSEARCH,
+  INDEX_PATTERN_KIBANA,
+  INDEX_PATTERN_LOGSTASH,
+  INDEX_PATTERN_BEATS,
+} from '../../../../common/constants';
+import { fetchStackProductUsage } from './fetch_stack_product_usage';
+import { getCcsIndexPattern } from '../../../lib/alerts/get_ccs_index_pattern';
+
+export const getStackProductsUsage = async (
+  config: MonitoringConfig,
+  callCluster: LegacyAPICaller,
+  availableCcs: string[],
+  clusterUuid: string
+): Promise<
+  Pick<
+    MonitoringClusterStackProductUsage,
+    'elasticsearch' | 'kibana' | 'logstash' | 'beats' | 'apm'
+  >
+> => {
+  const elasticsearchIndex = getCcsIndexPattern(INDEX_PATTERN_ELASTICSEARCH, availableCcs);
+  const kibanaIndex = getCcsIndexPattern(INDEX_PATTERN_KIBANA, availableCcs);
+  const logstashIndex = getCcsIndexPattern(INDEX_PATTERN_LOGSTASH, availableCcs);
+  const beatsIndex = getCcsIndexPattern(INDEX_PATTERN_BEATS, availableCcs);
+  const [elasticsearch, kibana, logstash, beats, apm] = await Promise.all([
+    fetchESUsage(config, callCluster, clusterUuid, elasticsearchIndex),
+    fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      kibanaIndex,
+      'kibana_stats',
+      'kibana_stats.kibana.uuid'
+    ),
+    fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      logstashIndex,
+      'logstash_stats',
+      'logstash_stats.logstash.uuid'
+    ),
+    fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      beatsIndex,
+      'beats_stats',
+      'beats_stats.beat.uuid'
+    ),
+    fetchStackProductUsage(
+      config,
+      callCluster,
+      clusterUuid,
+      beatsIndex,
+      'beats_stats',
+      'beats_stats.beat.uuid',
+      [{ term: { 'beats_stats.beat.type': 'apm-server' } }]
+    ),
+  ]);
+
+  return { elasticsearch, kibana, logstash, beats, apm };
+};
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/types.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/types.ts
new file mode 100644
index 00000000000000..c8e0eeea815e15
--- /dev/null
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/types.ts
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface MonitoringUsage {
+  hasMonitoringData: boolean;
+  clusters: MonitoringClusterStackProductUsage[];
+}
+
+export interface MonitoringClusterStackProductUsage {
+  clusterUuid: string;
+  license: string;
+  metricbeatUsed: boolean;
+  elasticsearch: StackProductUsage;
+  logstash: StackProductUsage;
+  kibana: StackProductUsage;
+  beats: StackProductUsage;
+  apm: StackProductUsage;
+}
+
+export interface StackProductUsage {
+  count: number;
+  enabled: boolean;
+  metricbeatUsed: boolean;
+}
diff --git a/x-pack/plugins/monitoring/server/lib/alerts/get_ccs_index_pattern.ts b/x-pack/plugins/monitoring/server/lib/alerts/get_ccs_index_pattern.ts
index 1907d2b4b34015..f16e463b508fba 100644
--- a/x-pack/plugins/monitoring/server/lib/alerts/get_ccs_index_pattern.ts
+++ b/x-pack/plugins/monitoring/server/lib/alerts/get_ccs_index_pattern.ts
@@ -4,6 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 export function getCcsIndexPattern(indexPattern: string, remotes: string[]): string {
+  if (remotes.length === 0) {
+    return indexPattern;
+  }
   const patternsToAdd = [];
   for (const index of indexPattern.split(',')) {
     for (const remote of remotes) {
diff --git a/x-pack/plugins/monitoring/server/plugin.test.ts b/x-pack/plugins/monitoring/server/plugin.test.ts
index a2520593c436d8..727ada52e6e3dc 100644
--- a/x-pack/plugins/monitoring/server/plugin.test.ts
+++ b/x-pack/plugins/monitoring/server/plugin.test.ts
@@ -14,7 +14,9 @@ jest.mock('rxjs', () => ({
 }));
 
 jest.mock('./es_client/instantiate_client', () => ({
-  instantiateClient: jest.fn(),
+  instantiateClient: jest.fn().mockImplementation(() => ({
+    cluster: {},
+  })),
 }));
 
 jest.mock('./license_service', () => ({
@@ -25,6 +27,10 @@ jest.mock('./license_service', () => ({
   })),
 }));
 
+jest.mock('./kibana_monitoring/collectors', () => ({
+  registerCollectors: jest.fn(),
+}));
+
 describe('Monitoring plugin', () => {
   const initializerContext = {
     logger: {
@@ -70,6 +76,9 @@ describe('Monitoring plugin', () => {
         subscribe: jest.fn(),
       },
     },
+    savedObjects: {
+      registerType: jest.fn(),
+    },
   };
 
   const setupPlugins = {
diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts
index 8d3248ddf43697..d8c1ff15a11998 100644
--- a/x-pack/plugins/monitoring/server/plugin.ts
+++ b/x-pack/plugins/monitoring/server/plugin.ts
@@ -28,6 +28,7 @@ import {
   KIBANA_MONITORING_LOGGING_TAG,
   KIBANA_STATS_TYPE_MONITORING,
   ALERTS,
+  SAVED_OBJECT_TELEMETRY,
 } from '../common/constants';
 import { MonitoringConfig, createConfig, configSchema } from './config';
 // @ts-ignore
@@ -157,7 +158,20 @@ export class Plugin {
 
     // Register collector objects for stats to show up in the APIs
     if (plugins.usageCollection) {
-      registerCollectors(plugins.usageCollection, config);
+      core.savedObjects.registerType({
+        name: SAVED_OBJECT_TELEMETRY,
+        hidden: true,
+        namespaceType: 'agnostic',
+        mappings: {
+          properties: {
+            reportedClusterUuids: {
+              type: 'keyword',
+            },
+          },
+        },
+      });
+
+      registerCollectors(plugins.usageCollection, config, cluster.callAsInternalUser);
     }
 
     // Always create the bulk uploader
diff --git a/x-pack/plugins/observability/public/typings/fetch_overview_data/index.ts b/x-pack/plugins/observability/public/typings/fetch_overview_data/index.ts
index 4dde78fb4cebb8..a87ae3fb261593 100644
--- a/x-pack/plugins/observability/public/typings/fetch_overview_data/index.ts
+++ b/x-pack/plugins/observability/public/typings/fetch_overview_data/index.ts
@@ -32,7 +32,10 @@ export type FetchData<T extends FetchDataResponse = FetchDataResponse> = (
 
 export type HasData = () => Promise<boolean>;
 
-export type ObservabilityFetchDataPlugins = Exclude<ObservabilityApp, 'observability'>;
+export type ObservabilityFetchDataPlugins = Exclude<
+  ObservabilityApp,
+  'observability' | 'stack_monitoring'
+>;
 
 export interface DataHandler<
   T extends ObservabilityFetchDataPlugins = ObservabilityFetchDataPlugins
diff --git a/x-pack/plugins/observability/typings/common.ts b/x-pack/plugins/observability/typings/common.ts
index c1b01c847f1645..845652031a5782 100644
--- a/x-pack/plugins/observability/typings/common.ts
+++ b/x-pack/plugins/observability/typings/common.ts
@@ -4,7 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export type ObservabilityApp = 'infra_metrics' | 'infra_logs' | 'apm' | 'uptime' | 'observability';
+export type ObservabilityApp =
+  | 'infra_metrics'
+  | 'infra_logs'
+  | 'apm'
+  | 'uptime'
+  | 'observability'
+  | 'stack_monitoring';
 
 export type PromiseReturnType<Func> = Func extends (...args: any[]) => Promise<infer Value>
   ? Value
diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
index 86b7889957c9fd..1236f2ad9b5596 100644
--- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
+++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
@@ -540,6 +540,91 @@
         }
       }
     },
+    "monitoring": {
+      "properties": {
+        "hasMonitoringData": {
+          "type": "boolean"
+        },
+        "clusters": {
+          "properties": {
+            "license": {
+              "type": "keyword"
+            },
+            "clusterUuid": {
+              "type": "keyword"
+            },
+            "metricbeatUsed": {
+              "type": "boolean"
+            },
+            "elasticsearch": {
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "metricbeatUsed": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "kibana": {
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "metricbeatUsed": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "logstash": {
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "metricbeatUsed": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "beats": {
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "metricbeatUsed": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "apm": {
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "metricbeatUsed": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "rollups": {
       "properties": {
         "index_patterns": {

From 0dc89cb7165a8ed3ec93ce8dadc51d93360ca3ae Mon Sep 17 00:00:00 2001
From: CJ Cenizal <cj@cenizal.com>
Date: Fri, 25 Sep 2020 08:25:49 -0700
Subject: [PATCH 12/22] Add support for runtime field types to mappings editor.
 (#77420)

* Add support for runtime field types to mappings editor.
* Add tests for getTypeLabelFromField util.
* Refine callout appearance in term vector and alias parameters.
---
 .../public/application/app_context.tsx        |  3 +-
 .../document_fields/field_parameters/index.ts |  4 +
 .../painless_script_parameter.tsx             | 80 ++++++++++++++++
 .../field_parameters/path_parameter.tsx       | 22 ++---
 .../runtime_type_parameter.tsx                | 96 +++++++++++++++++++
 .../term_vector_parameter.tsx                 | 15 +--
 .../fields/create_field/create_field.tsx      | 23 +++--
 .../required_parameters_forms/index.ts        |  4 +-
 .../runtime_type.tsx                          | 18 ++++
 .../edit_field/edit_field_header_form.tsx     | 66 ++++++++-----
 .../edit_field/field_description_section.tsx  |  3 +-
 .../fields/field_beta_badge.tsx               | 21 ++++
 .../fields/field_types/index.ts               |  2 +
 .../fields/field_types/runtime_type.tsx       | 19 ++++
 .../fields/fields_list_item.tsx               |  7 +-
 .../search_fields/search_result_item.tsx      |  4 +-
 .../constants/data_types_definition.tsx       | 20 ++++
 .../constants/field_options.tsx               | 30 ++++++
 .../constants/parameters_definition.tsx       | 48 ++++++++++
 .../components/mappings_editor/lib/index.ts   | 22 ++++-
 .../mappings_editor/lib/utils.test.ts         | 49 +++++++++-
 .../components/mappings_editor/lib/utils.ts   | 19 +++-
 .../mappings_editor/shared_imports.ts         |  2 +
 .../mappings_editor/types/document_fields.ts  |  9 +-
 .../public/application/index.tsx              | 33 ++++---
 .../application/mount_management_section.ts   |  2 +
 .../index_management/public/shared_imports.ts |  5 +-
 27 files changed, 552 insertions(+), 74 deletions(-)
 create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/painless_script_parameter.tsx
 create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/runtime_type_parameter.tsx
 create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/runtime_type.tsx
 create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_beta_badge.tsx
 create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/runtime_type.tsx

diff --git a/x-pack/plugins/index_management/public/application/app_context.tsx b/x-pack/plugins/index_management/public/application/app_context.tsx
index 6fbe177d24e066..22e6f09907d75d 100644
--- a/x-pack/plugins/index_management/public/application/app_context.tsx
+++ b/x-pack/plugins/index_management/public/application/app_context.tsx
@@ -8,7 +8,7 @@ import React, { createContext, useContext } from 'react';
 import { ScopedHistory } from 'kibana/public';
 import { ManagementAppMountParams } from 'src/plugins/management/public';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/public';
-import { CoreStart } from '../../../../../src/core/public';
+import { CoreSetup, CoreStart } from '../../../../../src/core/public';
 
 import { IngestManagerSetup } from '../../../ingest_manager/public';
 import { IndexMgmtMetricsType } from '../types';
@@ -34,6 +34,7 @@ export interface AppDependencies {
   };
   history: ScopedHistory;
   setBreadcrumbs: ManagementAppMountParams['setBreadcrumbs'];
+  uiSettings: CoreSetup['uiSettings'];
 }
 
 export const AppContextProvider = ({
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts
index b3bf0719489562..c47ea4a884111f 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts
@@ -73,6 +73,10 @@ export * from './meta_parameter';
 
 export * from './ignore_above_parameter';
 
+export { RuntimeTypeParameter } from './runtime_type_parameter';
+
+export { PainlessScriptParameter } from './painless_script_parameter';
+
 export const PARAMETER_SERIALIZERS = [relationsSerializer, dynamicSerializer];
 
 export const PARAMETER_DESERIALIZERS = [relationsDeserializer, dynamicDeserializer];
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/painless_script_parameter.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/painless_script_parameter.tsx
new file mode 100644
index 00000000000000..19746034b530ce
--- /dev/null
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/painless_script_parameter.tsx
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { EuiFormRow, EuiDescribedFormGroup } from '@elastic/eui';
+
+import { CodeEditor, UseField } from '../../../shared_imports';
+import { getFieldConfig } from '../../../lib';
+import { EditFieldFormRow } from '../fields/edit_field';
+
+interface Props {
+  stack?: boolean;
+}
+
+export const PainlessScriptParameter = ({ stack }: Props) => {
+  return (
+    <UseField path="script.source" config={getFieldConfig('script')}>
+      {(scriptField) => {
+        const error = scriptField.getErrorsMessages();
+        const isInvalid = error ? Boolean(error.length) : false;
+
+        const field = (
+          <EuiFormRow label={scriptField.label} error={error} isInvalid={isInvalid} fullWidth>
+            <CodeEditor
+              languageId="painless"
+              // 99% width allows the editor to resize horizontally. 100% prevents it from resizing.
+              width="99%"
+              height="400px"
+              value={scriptField.value as string}
+              onChange={scriptField.setValue}
+              options={{
+                fontSize: 12,
+                minimap: {
+                  enabled: false,
+                },
+                scrollBeyondLastLine: false,
+                wordWrap: 'on',
+                wrappingIndent: 'indent',
+                automaticLayout: true,
+              }}
+            />
+          </EuiFormRow>
+        );
+
+        const fieldTitle = i18n.translate('xpack.idxMgmt.mappingsEditor.painlessScript.title', {
+          defaultMessage: 'Emitted value',
+        });
+
+        const fieldDescription = i18n.translate(
+          'xpack.idxMgmt.mappingsEditor.painlessScript.description',
+          {
+            defaultMessage: 'Use emit() to define the value of this runtime field.',
+          }
+        );
+
+        if (stack) {
+          return (
+            <EditFieldFormRow title={fieldTitle} description={fieldDescription} withToggle={false}>
+              {field}
+            </EditFieldFormRow>
+          );
+        }
+
+        return (
+          <EuiDescribedFormGroup
+            title={<h3>{fieldTitle}</h3>}
+            description={fieldDescription}
+            fullWidth={true}
+          >
+            {field}
+          </EuiDescribedFormGroup>
+        );
+      }}
+    </UseField>
+  );
+};
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/path_parameter.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/path_parameter.tsx
index 6575fe1fac7b85..62810df44b5af4 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/path_parameter.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/path_parameter.tsx
@@ -93,17 +93,17 @@ export const PathParameter = ({ field, allFields }: Props) => {
             <>
               {!Boolean(suggestedFields.length) && (
                 <>
-                  <EuiCallOut color="warning">
-                    <p>
-                      {i18n.translate(
-                        'xpack.idxMgmt.mappingsEditor.aliasType.noFieldsAddedWarningMessage',
-                        {
-                          defaultMessage:
-                            'You need to add at least one field before creating an alias.',
-                        }
-                      )}
-                    </p>
-                  </EuiCallOut>
+                  <EuiCallOut
+                    size="s"
+                    color="warning"
+                    title={i18n.translate(
+                      'xpack.idxMgmt.mappingsEditor.aliasType.noFieldsAddedWarningMessage',
+                      {
+                        defaultMessage:
+                          'You need to add at least one field before creating an alias.',
+                      }
+                    )}
+                  />
                   <EuiSpacer />
                 </>
               )}
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/runtime_type_parameter.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/runtime_type_parameter.tsx
new file mode 100644
index 00000000000000..4bdb15af5e7d99
--- /dev/null
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/runtime_type_parameter.tsx
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import {
+  EuiFormRow,
+  EuiComboBox,
+  EuiComboBoxOptionOption,
+  EuiDescribedFormGroup,
+  EuiSpacer,
+} from '@elastic/eui';
+
+import { UseField } from '../../../shared_imports';
+import { DataType } from '../../../types';
+import { getFieldConfig } from '../../../lib';
+import { RUNTIME_FIELD_OPTIONS, TYPE_DEFINITION } from '../../../constants';
+import { EditFieldFormRow, FieldDescriptionSection } from '../fields/edit_field';
+
+interface Props {
+  stack?: boolean;
+}
+
+export const RuntimeTypeParameter = ({ stack }: Props) => {
+  return (
+    <UseField path="runtime_type" config={getFieldConfig('runtime_type')}>
+      {(runtimeTypeField) => {
+        const { label, value, setValue } = runtimeTypeField;
+        const typeDefinition =
+          TYPE_DEFINITION[(value as EuiComboBoxOptionOption[])[0]!.value as DataType];
+
+        const field = (
+          <>
+            <EuiFormRow label={label} fullWidth>
+              <EuiComboBox
+                placeholder={i18n.translate(
+                  'xpack.idxMgmt.mappingsEditor.runtimeType.placeholderLabel',
+                  {
+                    defaultMessage: 'Select a type',
+                  }
+                )}
+                singleSelection={{ asPlainText: true }}
+                options={RUNTIME_FIELD_OPTIONS}
+                selectedOptions={value as EuiComboBoxOptionOption[]}
+                onChange={setValue}
+                isClearable={false}
+                fullWidth
+              />
+            </EuiFormRow>
+
+            <EuiSpacer size="m" />
+
+            {/* Field description */}
+            {typeDefinition && (
+              <FieldDescriptionSection isMultiField={false}>
+                {typeDefinition.description?.() as JSX.Element}
+              </FieldDescriptionSection>
+            )}
+          </>
+        );
+
+        const fieldTitle = i18n.translate('xpack.idxMgmt.mappingsEditor.runtimeType.title', {
+          defaultMessage: 'Emitted type',
+        });
+
+        const fieldDescription = i18n.translate(
+          'xpack.idxMgmt.mappingsEditor.runtimeType.description',
+          {
+            defaultMessage: 'Select the type of value emitted by the runtime field.',
+          }
+        );
+
+        if (stack) {
+          return (
+            <EditFieldFormRow title={fieldTitle} description={fieldDescription} withToggle={false}>
+              {field}
+            </EditFieldFormRow>
+          );
+        }
+
+        return (
+          <EuiDescribedFormGroup
+            title={<h3>{fieldTitle}</h3>}
+            description={fieldDescription}
+            fullWidth={true}
+          >
+            {field}
+          </EuiDescribedFormGroup>
+        );
+      }}
+    </UseField>
+  );
+};
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/term_vector_parameter.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/term_vector_parameter.tsx
index 6752bb6d6af2b1..69d56032eed6af 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/term_vector_parameter.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/term_vector_parameter.tsx
@@ -56,14 +56,17 @@ export const TermVectorParameter = ({ field, defaultToggleValue }: Props) => {
             {formData.term_vector === 'with_positions_offsets' && (
               <>
                 <EuiSpacer size="s" />
-                <EuiCallOut color="warning">
-                  <p>
-                    {i18n.translate('xpack.idxMgmt.mappingsEditor.termVectorFieldWarningMessage', {
+                <EuiCallOut
+                  size="s"
+                  color="warning"
+                  title={i18n.translate(
+                    'xpack.idxMgmt.mappingsEditor.termVectorFieldWarningMessage',
+                    {
                       defaultMessage:
                         'Setting "With positions and offsets" will double the size of a field’s index.',
-                    })}
-                  </p>
-                </EuiCallOut>
+                    }
+                  )}
+                />
               </>
             )}
           </>
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/create_field.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/create_field.tsx
index a8170b1d59fbb9..cff2b9af4fd10d 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/create_field.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/create_field.tsx
@@ -14,15 +14,17 @@ import {
   EuiFlexGroup,
   EuiFlexItem,
   EuiOutsideClickDetector,
+  EuiSpacer,
 } from '@elastic/eui';
 
 import { useForm, Form, FormDataProvider } from '../../../../shared_imports';
-import { EUI_SIZE } from '../../../../constants';
+import { EUI_SIZE, TYPE_DEFINITION } from '../../../../constants';
 import { useDispatch } from '../../../../mappings_state_context';
 import { fieldSerializer } from '../../../../lib';
-import { Field, NormalizedFields } from '../../../../types';
+import { Field, NormalizedFields, MainType } from '../../../../types';
 import { NameParameter, TypeParameter, SubTypeParameter } from '../../field_parameters';
-import { getParametersFormForType } from './required_parameters_forms';
+import { FieldBetaBadge } from '../field_beta_badge';
+import { getRequiredParametersFormForType } from './required_parameters_forms';
 
 const formWrapper = (props: any) => <form {...props} />;
 
@@ -195,18 +197,27 @@ export const CreateField = React.memo(function CreateFieldComponent({
 
             <FormDataProvider pathsToWatch={['type', 'subType']}>
               {({ type, subType }) => {
-                const ParametersForm = getParametersFormForType(
+                const RequiredParametersForm = getRequiredParametersFormForType(
                   type?.[0].value,
                   subType?.[0].value
                 );
 
-                if (!ParametersForm) {
+                if (!RequiredParametersForm) {
                   return null;
                 }
 
+                const typeDefinition = TYPE_DEFINITION[type?.[0].value as MainType];
+
                 return (
                   <div className="mappingsEditor__createFieldRequiredProps">
-                    <ParametersForm key={subType ?? type} allFields={allFields} />
+                    {typeDefinition.isBeta ? (
+                      <>
+                        <FieldBetaBadge />
+                        <EuiSpacer size="m" />
+                      </>
+                    ) : null}
+
+                    <RequiredParametersForm key={subType ?? type} allFields={allFields} />
                   </div>
                 );
               }}
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/index.ts
index 1112bf99713ed1..5c04b2fbb336c5 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/index.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/index.ts
@@ -11,6 +11,7 @@ import { AliasTypeRequiredParameters } from './alias_type';
 import { TokenCountTypeRequiredParameters } from './token_count_type';
 import { ScaledFloatTypeRequiredParameters } from './scaled_float_type';
 import { DenseVectorRequiredParameters } from './dense_vector_type';
+import { RuntimeTypeRequiredParameters } from './runtime_type';
 
 export interface ComponentProps {
   allFields: NormalizedFields['byId'];
@@ -21,9 +22,10 @@ const typeToParametersFormMap: { [key in DataType]?: ComponentType<any> } = {
   token_count: TokenCountTypeRequiredParameters,
   scaled_float: ScaledFloatTypeRequiredParameters,
   dense_vector: DenseVectorRequiredParameters,
+  runtime: RuntimeTypeRequiredParameters,
 };
 
-export const getParametersFormForType = (
+export const getRequiredParametersFormForType = (
   type: MainType,
   subType?: SubType
 ): ComponentType<ComponentProps> | undefined =>
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/runtime_type.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/runtime_type.tsx
new file mode 100644
index 00000000000000..54907295f8a157
--- /dev/null
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/required_parameters_forms/runtime_type.tsx
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { RuntimeTypeParameter, PainlessScriptParameter } from '../../../field_parameters';
+
+export const RuntimeTypeRequiredParameters = () => {
+  return (
+    <>
+      <RuntimeTypeParameter />
+      <PainlessScriptParameter />
+    </>
+  );
+};
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/edit_field_header_form.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/edit_field_header_form.tsx
index 3b55c5ac076c29..e91a666cc42219 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/edit_field_header_form.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/edit_field_header_form.tsx
@@ -5,12 +5,13 @@
  */
 
 import React from 'react';
-import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui';
 
 import { FormDataProvider } from '../../../../shared_imports';
-import { MainType, SubType, Field } from '../../../../types';
+import { MainType, SubType, Field, DataTypeDefinition } from '../../../../types';
 import { TYPE_DEFINITION } from '../../../../constants';
 import { NameParameter, TypeParameter, SubTypeParameter } from '../../field_parameters';
+import { FieldBetaBadge } from '../field_beta_badge';
 import { FieldDescriptionSection } from './field_description_section';
 
 interface Props {
@@ -19,6 +20,25 @@ interface Props {
   isMultiField: boolean;
 }
 
+const getTypeDefinition = (type: MainType, subType: SubType): DataTypeDefinition | undefined => {
+  if (!type) {
+    return;
+  }
+
+  const typeDefinition = TYPE_DEFINITION[type];
+  const hasSubType = typeDefinition.subTypes !== undefined;
+
+  if (hasSubType) {
+    if (subType) {
+      return TYPE_DEFINITION[subType];
+    }
+
+    return;
+  }
+
+  return typeDefinition;
+};
+
 export const EditFieldHeaderForm = React.memo(
   ({ defaultValue, isRootLevelField, isMultiField }: Props) => {
     return (
@@ -56,27 +76,29 @@ export const EditFieldHeaderForm = React.memo(
         </EuiFlexGroup>
 
         {/* Field description */}
-        <FieldDescriptionSection isMultiField={isMultiField}>
-          <FormDataProvider pathsToWatch={['type', 'subType']}>
-            {({ type, subType }) => {
-              if (!type) {
-                return null;
-              }
-              const typeDefinition = TYPE_DEFINITION[type[0].value as MainType];
-              const hasSubType = typeDefinition.subTypes !== undefined;
-
-              if (hasSubType) {
-                if (subType) {
-                  const subTypeDefinition = TYPE_DEFINITION[subType as SubType];
-                  return (subTypeDefinition?.description?.() as JSX.Element) ?? null;
-                }
-                return null;
-              }
+        <FormDataProvider pathsToWatch={['type', 'subType']}>
+          {({ type, subType }) => {
+            const typeDefinition = getTypeDefinition(
+              type[0].value as MainType,
+              subType && (subType[0].value as SubType)
+            );
+            const description = (typeDefinition?.description?.() as JSX.Element) ?? null;
 
-              return typeDefinition.description?.() as JSX.Element;
-            }}
-          </FormDataProvider>
-        </FieldDescriptionSection>
+            return (
+              <>
+                <EuiSpacer size="l" />
+
+                {typeDefinition?.isBeta && <FieldBetaBadge />}
+
+                <EuiSpacer size="s" />
+
+                <FieldDescriptionSection isMultiField={isMultiField}>
+                  {description}
+                </FieldDescriptionSection>
+              </>
+            );
+          }}
+        </FormDataProvider>
       </>
     );
   }
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/field_description_section.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/field_description_section.tsx
index 2040d7f40d5cb2..2301f7a47bf2ff 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/field_description_section.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/edit_field/field_description_section.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import React from 'react';
-import { EuiSpacer, EuiText } from '@elastic/eui';
+import { EuiText } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 
 interface Props {
@@ -19,7 +19,6 @@ export const FieldDescriptionSection = ({ children, isMultiField }: Props) => {
 
   return (
     <section>
-      <EuiSpacer size="l" />
       <EuiText size="s" color="subdued">
         {children}
 
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_beta_badge.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_beta_badge.tsx
new file mode 100644
index 00000000000000..99c725e8a00b37
--- /dev/null
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_beta_badge.tsx
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiBetaBadge } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+export const FieldBetaBadge = () => {
+  const betaText = i18n.translate('xpack.idxMgmt.mappingsEditor.fieldBetaBadgeLabel', {
+    defaultMessage: 'Beta',
+  });
+
+  const tooltipText = i18n.translate('xpack.idxMgmt.mappingsEditor.fieldBetaBadgeTooltip', {
+    defaultMessage: 'This field type is not GA. Please help us by reporting any bugs.',
+  });
+
+  return <EuiBetaBadge label={betaText} tooltipContent={tooltipText} />;
+};
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts
index 6b092c5561b3bf..4d36b4dd2578d8 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts
@@ -31,6 +31,7 @@ import { JoinType } from './join_type';
 import { HistogramType } from './histogram_type';
 import { ConstantKeywordType } from './constant_keyword_type';
 import { RankFeatureType } from './rank_feature_type';
+import { RuntimeType } from './runtime_type';
 import { WildcardType } from './wildcard_type';
 import { PointType } from './point_type';
 
@@ -60,6 +61,7 @@ const typeToParametersFormMap: { [key in DataType]?: ComponentType<any> } = {
   histogram: HistogramType,
   constant_keyword: ConstantKeywordType,
   rank_feature: RankFeatureType,
+  runtime: RuntimeType,
   wildcard: WildcardType,
   point: PointType,
 };
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/runtime_type.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/runtime_type.tsx
new file mode 100644
index 00000000000000..dcf5a74e0e304d
--- /dev/null
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/runtime_type.tsx
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { RuntimeTypeParameter, PainlessScriptParameter } from '../../field_parameters';
+import { BasicParametersSection } from '../edit_field';
+
+export const RuntimeType = () => {
+  return (
+    <BasicParametersSection>
+      <RuntimeTypeParameter stack={true} />
+      <PainlessScriptParameter stack={true} />
+    </BasicParametersSection>
+  );
+};
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/fields_list_item.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/fields_list_item.tsx
index 4ab0ea0fb355b6..1939f09fa6762b 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/fields_list_item.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/fields_list_item.tsx
@@ -16,7 +16,7 @@ import {
 import { i18n } from '@kbn/i18n';
 
 import { NormalizedField, NormalizedFields } from '../../../types';
-import { getTypeLabelFromType } from '../../../lib';
+import { getTypeLabelFromField } from '../../../lib';
 import { CHILD_FIELD_INDENT_SIZE, LEFT_PADDING_SIZE_FIELD_ITEM_WRAPPER } from '../../../constants';
 
 import { FieldsList } from './fields_list';
@@ -67,6 +67,7 @@ function FieldListItemComponent(
     isExpanded,
     path,
   } = field;
+
   // When there aren't any "child" fields (the maxNestedDepth === 0), there is no toggle icon on the left of any field.
   // For that reason, we need to compensate and substract some indent to left align on the page.
   const substractIndentAmount = maxNestedDepth === 0 ? CHILD_FIELD_INDENT_SIZE * 0.5 : 0;
@@ -280,10 +281,10 @@ function FieldListItemComponent(
                   ? i18n.translate('xpack.idxMgmt.mappingsEditor.multiFieldBadgeLabel', {
                       defaultMessage: '{dataType} multi-field',
                       values: {
-                        dataType: getTypeLabelFromType(source.type),
+                        dataType: getTypeLabelFromField(source),
                       },
                     })
-                  : getTypeLabelFromType(source.type)}
+                  : getTypeLabelFromField(source)}
               </EuiBadge>
             </EuiFlexItem>
 
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/search_fields/search_result_item.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/search_fields/search_result_item.tsx
index a2d9a50f283949..a4cc4b4776e2bf 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/search_fields/search_result_item.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/search_fields/search_result_item.tsx
@@ -11,7 +11,7 @@ import { i18n } from '@kbn/i18n';
 import { SearchResult } from '../../../types';
 import { TYPE_DEFINITION } from '../../../constants';
 import { useDispatch } from '../../../mappings_state_context';
-import { getTypeLabelFromType } from '../../../lib';
+import { getTypeLabelFromField } from '../../../lib';
 import { DeleteFieldProvider } from '../fields/delete_field_provider';
 
 interface Props {
@@ -121,7 +121,7 @@ export const SearchResultItem = React.memo(function FieldListItemFlatComponent({
                         dataType: TYPE_DEFINITION[source.type].label,
                       },
                     })
-                  : getTypeLabelFromType(source.type)}
+                  : getTypeLabelFromField(source)}
               </EuiBadge>
             </EuiFlexItem>
 
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx
index 293ae56d57ace5..7bcd8f32f1a7d8 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx
@@ -13,6 +13,25 @@ import { documentationService } from '../../../services/documentation';
 import { MainType, SubType, DataType, DataTypeDefinition } from '../types';
 
 export const TYPE_DEFINITION: { [key in DataType]: DataTypeDefinition } = {
+  runtime: {
+    value: 'runtime',
+    isBeta: true,
+    label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.runtimeFieldDescription', {
+      defaultMessage: 'Runtime',
+    }),
+    // TODO: Add this once the page exists.
+    // documentation: {
+    //   main: '/runtime_field.html',
+    // },
+    description: () => (
+      <p>
+        <FormattedMessage
+          id="xpack.idxMgmt.mappingsEditor.dataType.runtimeFieldLongDescription"
+          defaultMessage="Runtime fields define scripts that calculate field values at runtime."
+        />
+      </p>
+    ),
+  },
   text: {
     value: 'text',
     label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.textDescription', {
@@ -896,6 +915,7 @@ export const MAIN_TYPES: MainType[] = [
   'range',
   'rank_feature',
   'rank_features',
+  'runtime',
   'search_as_you_type',
   'shape',
   'text',
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/field_options.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/field_options.tsx
index d16bf68b80e5d8..25fdac5089b862 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/field_options.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/field_options.tsx
@@ -18,6 +18,7 @@ export const TYPE_NOT_ALLOWED_MULTIFIELD: DataType[] = [
   'object',
   'nested',
   'alias',
+  'runtime',
 ];
 
 export const FIELD_TYPES_OPTIONS = Object.entries(MAIN_DATA_TYPE_DEFINITION).map(
@@ -27,6 +28,35 @@ export const FIELD_TYPES_OPTIONS = Object.entries(MAIN_DATA_TYPE_DEFINITION).map
   })
 ) as ComboBoxOption[];
 
+export const RUNTIME_FIELD_OPTIONS = [
+  {
+    label: 'Keyword',
+    value: 'keyword',
+  },
+  {
+    label: 'Long',
+    value: 'long',
+  },
+  {
+    label: 'Double',
+    value: 'double',
+  },
+  {
+    label: 'Date',
+    value: 'date',
+  },
+  {
+    label: 'IP',
+    value: 'ip',
+  },
+  {
+    label: 'Boolean',
+    value: 'boolean',
+  },
+] as ComboBoxOption[];
+
+export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
+
 interface SuperSelectOptionConfig {
   inputDisplay: string;
   dropdownDisplay: JSX.Element;
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/parameters_definition.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/parameters_definition.tsx
index 4ffedc8ca114d9..4c9786d88bfa2d 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/parameters_definition.tsx
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/parameters_definition.tsx
@@ -20,6 +20,7 @@ import {
 import {
   AliasOption,
   DataType,
+  RuntimeType,
   ComboBoxOption,
   ParameterName,
   ParameterDefinition,
@@ -27,6 +28,7 @@ import {
 import { documentationService } from '../../../services/documentation';
 import { INDEX_DEFAULT } from './default_values';
 import { TYPE_DEFINITION } from './data_types_definition';
+import { RUNTIME_FIELD_OPTIONS } from './field_options';
 
 const { toInt } = fieldFormatters;
 const { emptyField, containsCharsField, numberGreaterThanField, isJsonField } = fieldValidators;
@@ -185,6 +187,52 @@ export const PARAMETERS_DEFINITION: { [key in ParameterName]: ParameterDefinitio
     },
     schema: t.string,
   },
+  runtime_type: {
+    fieldConfig: {
+      type: FIELD_TYPES.COMBO_BOX,
+      label: i18n.translate('xpack.idxMgmt.mappingsEditor.parameters.runtimeTypeLabel', {
+        defaultMessage: 'Type',
+      }),
+      defaultValue: 'keyword',
+      deserializer: (fieldType: RuntimeType | undefined) => {
+        if (typeof fieldType === 'string' && Boolean(fieldType)) {
+          const label =
+            RUNTIME_FIELD_OPTIONS.find(({ value }) => value === fieldType)?.label ?? fieldType;
+          return [
+            {
+              label,
+              value: fieldType,
+            },
+          ];
+        }
+        return [];
+      },
+      serializer: (value: ComboBoxOption[]) => (value.length === 0 ? '' : value[0].value),
+    },
+    schema: t.string,
+  },
+  script: {
+    fieldConfig: {
+      defaultValue: '',
+      type: FIELD_TYPES.TEXT,
+      label: i18n.translate('xpack.idxMgmt.mappingsEditor.parameters.painlessScriptLabel', {
+        defaultMessage: 'Script',
+      }),
+      validations: [
+        {
+          validator: emptyField(
+            i18n.translate(
+              'xpack.idxMgmt.mappingsEditor.parameters.validations.scriptIsRequiredErrorMessage',
+              {
+                defaultMessage: 'Script must emit() a value.',
+              }
+            )
+          ),
+        },
+      ],
+    },
+    schema: t.string,
+  },
   store: {
     fieldConfig: {
       type: FIELD_TYPES.CHECKBOX,
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/index.ts
index 8cd1bbf0764ab0..0a59cafdcef47c 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/index.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/index.ts
@@ -4,7 +4,27 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export * from './utils';
+export {
+  getUniqueId,
+  getChildFieldsName,
+  getFieldMeta,
+  getTypeLabelFromField,
+  getFieldConfig,
+  getTypeMetaFromSource,
+  normalize,
+  deNormalize,
+  updateFieldsPathAfterFieldNameChange,
+  getAllChildFields,
+  getAllDescendantAliases,
+  getFieldAncestors,
+  filterTypesForMultiField,
+  filterTypesForNonRootFields,
+  getMaxNestedDepth,
+  buildFieldTreeFromIds,
+  shouldDeleteChildFieldsAfterTypeChange,
+  canUseMappingsEditor,
+  stripUndefinedValues,
+} from './utils';
 
 export * from './serializers';
 
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.test.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.test.ts
index bc495b05e07b70..e1988c071314ee 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.test.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.test.ts
@@ -4,9 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-jest.mock('../constants', () => ({ MAIN_DATA_TYPE_DEFINITION: {} }));
+jest.mock('../constants', () => {
+  const { TYPE_DEFINITION } = jest.requireActual('../constants');
+  return { MAIN_DATA_TYPE_DEFINITION: {}, TYPE_DEFINITION };
+});
 
-import { stripUndefinedValues } from './utils';
+import { stripUndefinedValues, getTypeLabelFromField } from './utils';
 
 describe('utils', () => {
   describe('stripUndefinedValues()', () => {
@@ -53,4 +56,46 @@ describe('utils', () => {
       expect(stripUndefinedValues(dataIN)).toEqual(dataOUT);
     });
   });
+
+  describe('getTypeLabelFromField()', () => {
+    test('returns an unprocessed label for non-runtime fields', () => {
+      expect(
+        getTypeLabelFromField({
+          name: 'testField',
+          type: 'keyword',
+        })
+      ).toBe('Keyword');
+    });
+
+    test(`returns a label prepended with 'Other' for unrecognized fields`, () => {
+      expect(
+        getTypeLabelFromField({
+          name: 'testField',
+          // @ts-ignore
+          type: 'hyperdrive',
+        })
+      ).toBe('Other: hyperdrive');
+    });
+
+    test("returns a label prepended with 'Runtime' for runtime fields", () => {
+      expect(
+        getTypeLabelFromField({
+          name: 'testField',
+          type: 'runtime',
+          runtime_type: 'keyword',
+        })
+      ).toBe('Runtime Keyword');
+    });
+
+    test("returns a label prepended with 'Runtime Other' for unrecognized runtime fields", () => {
+      expect(
+        getTypeLabelFromField({
+          name: 'testField',
+          type: 'runtime',
+          // @ts-ignore
+          runtime_type: 'hyperdrive',
+        })
+      ).toBe('Runtime Other: hyperdrive');
+    });
+  });
 });
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.ts
index 8b3ff600053054..d96f20683216a3 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/lib/utils.ts
@@ -71,8 +71,23 @@ export const getFieldMeta = (field: Field, isMultiField?: boolean): FieldMeta =>
   };
 };
 
-export const getTypeLabelFromType = (type: DataType) =>
-  TYPE_DEFINITION[type] ? TYPE_DEFINITION[type].label : `${TYPE_DEFINITION.other.label}: ${type}`;
+const getTypeLabel = (type?: DataType): string => {
+  return type && TYPE_DEFINITION[type]
+    ? TYPE_DEFINITION[type].label
+    : `${TYPE_DEFINITION.other.label}: ${type}`;
+};
+
+export const getTypeLabelFromField = (field: Field) => {
+  const { type, runtime_type: runtimeType } = field;
+  const typeLabel = getTypeLabel(type);
+
+  if (type === 'runtime') {
+    const runtimeTypeLabel = getTypeLabel(runtimeType);
+    return `${typeLabel} ${runtimeTypeLabel}`;
+  }
+
+  return typeLabel;
+};
 
 export const getFieldConfig = <T = unknown>(
   param: ParameterName,
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/shared_imports.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/shared_imports.ts
index 097d0395279502..54b24861081833 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/shared_imports.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/shared_imports.ts
@@ -51,3 +51,5 @@ export {
   OnJsonEditorUpdateHandler,
   GlobalFlyout,
 } from '../../../../../../../src/plugins/es_ui_shared/public';
+
+export { CodeEditor } from '../../../../../../../src/plugins/kibana_react/public';
diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts
index ca38a8d1e6c33c..48282abd1d799b 100644
--- a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts
+++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts
@@ -8,7 +8,7 @@ import { ReactNode } from 'react';
 import { GenericObject } from './mappings_editor';
 
 import { FieldConfig } from '../shared_imports';
-import { PARAMETERS_DEFINITION } from '../constants';
+import { PARAMETERS_DEFINITION, RUNTIME_FIELD_TYPES } from '../constants';
 
 export interface DataTypeDefinition {
   label: string;
@@ -19,6 +19,7 @@ export interface DataTypeDefinition {
   };
   subTypes?: { label: string; types: SubType[] };
   description?: () => ReactNode;
+  isBeta?: boolean;
 }
 
 export interface ParameterDefinition {
@@ -35,6 +36,7 @@ export interface ParameterDefinition {
 }
 
 export type MainType =
+  | 'runtime'
   | 'text'
   | 'keyword'
   | 'numeric'
@@ -73,6 +75,8 @@ export type SubType = NumericType | RangeType;
 
 export type DataType = MainType | SubType;
 
+export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
+
 export type NumericType =
   | 'long'
   | 'integer'
@@ -151,6 +155,8 @@ export type ParameterName =
   | 'depth_limit'
   | 'relations'
   | 'max_shingle_size'
+  | 'runtime_type'
+  | 'script'
   | 'value'
   | 'meta';
 
@@ -168,6 +174,7 @@ export interface Fields {
 interface FieldBasic {
   name: string;
   type: DataType;
+  runtime_type?: DataType;
   subType?: SubType;
   properties?: { [key: string]: Omit<Field, 'name'> };
   fields?: { [key: string]: Omit<Field, 'name'> };
diff --git a/x-pack/plugins/index_management/public/application/index.tsx b/x-pack/plugins/index_management/public/application/index.tsx
index f881c2e01cefc0..d8b5da8361c435 100644
--- a/x-pack/plugins/index_management/public/application/index.tsx
+++ b/x-pack/plugins/index_management/public/application/index.tsx
@@ -11,7 +11,7 @@ import { render, unmountComponentAtNode } from 'react-dom';
 import { CoreStart } from '../../../../../src/core/public';
 
 import { API_BASE_PATH } from '../../common';
-import { GlobalFlyout } from '../shared_imports';
+import { createKibanaReactContext, GlobalFlyout } from '../shared_imports';
 
 import { AppContextProvider, AppDependencies } from './app_context';
 import { App } from './app';
@@ -30,7 +30,12 @@ export const renderApp = (
 
   const { i18n, docLinks, notifications, application } = core;
   const { Context: I18nContext } = i18n;
-  const { services, history, setBreadcrumbs } = dependencies;
+  const { services, history, setBreadcrumbs, uiSettings } = dependencies;
+
+  // uiSettings is required by the CodeEditor component used to edit runtime field Painless scripts.
+  const { Provider: KibanaReactContextProvider } = createKibanaReactContext({
+    uiSettings,
+  });
 
   const componentTemplateProviderValues = {
     httpClient: services.httpService.httpClient,
@@ -44,17 +49,19 @@ export const renderApp = (
 
   render(
     <I18nContext>
-      <Provider store={indexManagementStore(services)}>
-        <AppContextProvider value={dependencies}>
-          <MappingsEditorProvider>
-            <ComponentTemplatesProvider value={componentTemplateProviderValues}>
-              <GlobalFlyoutProvider>
-                <App history={history} />
-              </GlobalFlyoutProvider>
-            </ComponentTemplatesProvider>
-          </MappingsEditorProvider>
-        </AppContextProvider>
-      </Provider>
+      <KibanaReactContextProvider>
+        <Provider store={indexManagementStore(services)}>
+          <AppContextProvider value={dependencies}>
+            <MappingsEditorProvider>
+              <ComponentTemplatesProvider value={componentTemplateProviderValues}>
+                <GlobalFlyoutProvider>
+                  <App history={history} />
+                </GlobalFlyoutProvider>
+              </ComponentTemplatesProvider>
+            </MappingsEditorProvider>
+          </AppContextProvider>
+        </Provider>
+      </KibanaReactContextProvider>
     </I18nContext>,
     elem
   );
diff --git a/x-pack/plugins/index_management/public/application/mount_management_section.ts b/x-pack/plugins/index_management/public/application/mount_management_section.ts
index 6257b68430cf07..f7b728c8757627 100644
--- a/x-pack/plugins/index_management/public/application/mount_management_section.ts
+++ b/x-pack/plugins/index_management/public/application/mount_management_section.ts
@@ -41,6 +41,7 @@ export async function mountManagementSection(
     fatalErrors,
     application,
     chrome: { docTitle },
+    uiSettings,
   } = core;
 
   docTitle.change(PLUGIN.getI18nName(i18n));
@@ -60,6 +61,7 @@ export async function mountManagementSection(
     services,
     history,
     setBreadcrumbs,
+    uiSettings,
   };
 
   const unmountAppCallback = renderApp(element, { core, dependencies: appDependencies });
diff --git a/x-pack/plugins/index_management/public/shared_imports.ts b/x-pack/plugins/index_management/public/shared_imports.ts
index d58545768732e1..acb3eb512e2c18 100644
--- a/x-pack/plugins/index_management/public/shared_imports.ts
+++ b/x-pack/plugins/index_management/public/shared_imports.ts
@@ -44,4 +44,7 @@ export {
 
 export { isJSON } from '../../../../src/plugins/es_ui_shared/static/validators/string';
 
-export { reactRouterNavigate } from '../../../../src/plugins/kibana_react/public';
+export {
+  createKibanaReactContext,
+  reactRouterNavigate,
+} from '../../../../src/plugins/kibana_react/public';

From cf18e4637e9a35ff5434bbd8162588eeef24d33e Mon Sep 17 00:00:00 2001
From: ncheckin <68351161+ncheckin@users.noreply.github.com>
Date: Fri, 25 Sep 2020 16:03:32 -0400
Subject: [PATCH 13/22] Update tutorial-define-index.asciidoc (#75754)

adds windows alternative for curl

Co-authored-by: Kaarina Tungseth <kaarina.tungseth@elastic.co>
---
 docs/getting-started/tutorial-define-index.asciidoc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/getting-started/tutorial-define-index.asciidoc b/docs/getting-started/tutorial-define-index.asciidoc
index fbe7450683dbc1..cb3f6c9ff0c9bf 100644
--- a/docs/getting-started/tutorial-define-index.asciidoc
+++ b/docs/getting-started/tutorial-define-index.asciidoc
@@ -47,5 +47,11 @@ contains the time series data.
 [role="screenshot"]
 image::images/tutorial_index_patterns.png[All tutorial index patterns]
 
+NOTE: When you define an index pattern, the indices that match that pattern must
+exist in Elasticsearch and they must contain data. To check if the indices are
+available, open the menu, go to *Dev Tools > Console*, then enter `GET _cat/indices`. Alternately, use
+`curl -XGET "http://localhost:9200/_cat/indices"`.
+For Windows, run `Invoke-RestMethod -Uri "http://localhost:9200/_cat/indices"` in Powershell.
+
 
 

From 500ad8baf038b1fe29c6b11cf305cdfa6f2c1542 Mon Sep 17 00:00:00 2001
From: ncheckin <68351161+ncheckin@users.noreply.github.com>
Date: Fri, 25 Sep 2020 16:03:57 -0400
Subject: [PATCH 14/22] Update tutorial-full-experience.asciidoc (#75836)

add powershell alternative for curl -O commands at beginning of doc

Co-authored-by: Kaarina Tungseth <kaarina.tungseth@elastic.co>
---
 docs/getting-started/tutorial-full-experience.asciidoc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/getting-started/tutorial-full-experience.asciidoc b/docs/getting-started/tutorial-full-experience.asciidoc
index 1e6fe39dbd013c..a7d5412ae06322 100644
--- a/docs/getting-started/tutorial-full-experience.asciidoc
+++ b/docs/getting-started/tutorial-full-experience.asciidoc
@@ -25,7 +25,14 @@ curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/shakespeare.
 curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/accounts.zip
 curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/logs.jsonl.gz
 
-Two of the data sets are compressed. To extract the files, use the following commands:
+Alternatively, for Windows users, run the following commands in Powershell:
+
+[source,shell]
+Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/shakespeare.json -OutFile shakespeare.json
+Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/accounts.zip -OutFile accounts.zip
+Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/logs.jsonl.gz -OutFile logs.jsonl.gz
+
+Two of the data sets are compressed. To extract the files, use these commands:
 
 [source,shell]
 unzip accounts.zip

From faf4b040042918fe804bc3f80e42011a073c2d8c Mon Sep 17 00:00:00 2001
From: Marshall Main <55718608+marshallmain@users.noreply.github.com>
Date: Fri, 25 Sep 2020 16:23:25 -0400
Subject: [PATCH 15/22] [Detections][EQL] EQL rule execution in detection
 engine (#77419)

* First draft of EQL rules in detection engine

* Reorganize functions to separate files

* Start adding eventCategoryOverride option for EQL rules

* Add building block alerts for each event within sequence

* Use eql instead of eql_query for rule type

* Remove unused imports

* Fix tests

* Add basic tests for buildEqlSearchRequest

* Add rulesSchema tests for eql

* Add buildSignalFromSequence test

* Add threat rule fields to buildRuleWithoutOverrides

* Fix buildSignalFromSequence typecheck error

* Add more tests

* Add tests for wrapBuildingBlock and generateSignalId

* Use isEqlRule function and fix import error

* delete frank

* Move sequence interface to types.ts

* Fix import

* Remove EQL execution placeholder, add back language to eql rule type

* allow no indices for eql search

* Fix unit tests for language update

* Fix buildEqlSearchRequest tests

* Replace signal.child with signal.group

* remove unused import

* Move sequence signal group building to separate testable function

* Unbork the merge conflict resolution

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../detection_engine/get_query_filter.test.ts | 136 +++++++++++-
 .../detection_engine/get_query_filter.ts      |  79 ++++++-
 .../schemas/common/schemas.ts                 |   6 +
 .../request/add_prepackaged_rules_schema.ts   |   2 +
 .../schemas/request/create_rules_schema.ts    |   2 +
 .../schemas/request/import_rules_schema.ts    |   2 +
 .../schemas/request/patch_rules_schema.ts     |   2 +
 .../schemas/request/update_rules_schema.ts    |   2 +
 .../schemas/response/rules_schema.mocks.ts    |  13 +-
 .../schemas/response/rules_schema.test.ts     |  33 ++-
 .../schemas/response/rules_schema.ts          |  23 +-
 .../routes/__mocks__/request_responses.ts     |   1 +
 .../routes/index/signals_mapping.json         |  10 +
 .../routes/rules/create_rules_bulk_route.ts   |   2 +
 .../routes/rules/create_rules_route.ts        |   7 +-
 .../routes/rules/import_rules_route.ts        |   3 +
 .../routes/rules/patch_rules_bulk_route.ts    |   2 +
 .../routes/rules/patch_rules_route.ts         |   2 +
 .../routes/rules/update_rules_bulk_route.ts   |   2 +
 .../routes/rules/update_rules_route.ts        |   2 +
 .../detection_engine/routes/rules/utils.ts    |   1 +
 .../rules/create_rules.mock.ts                |   2 +
 .../detection_engine/rules/create_rules.ts    |   2 +
 .../rules/install_prepacked_rules.ts          |   2 +
 .../rules/patch_rules.mock.ts                 |   2 +
 .../lib/detection_engine/rules/patch_rules.ts |   2 +
 .../lib/detection_engine/rules/types.ts       |   4 +
 .../rules/update_prepacked_rules.ts           |   2 +
 .../rules/update_rules.mock.ts                |   2 +
 .../detection_engine/rules/update_rules.ts    |   2 +
 .../lib/detection_engine/rules/utils.test.ts  |   3 +
 .../lib/detection_engine/rules/utils.ts       |   2 +
 .../scripts/rules/queries/query_eql.json      |  68 ++++++
 .../signals/__mocks__/es_results.ts           | 108 +++++++++-
 .../signals/build_bulk_body.test.ts           | 201 +++++++++++++++++-
 .../signals/build_bulk_body.ts                |  97 ++++++++-
 .../signals/build_event_type_signal.ts        |   4 +-
 .../signals/build_rule.test.ts                | 124 +++++++++--
 .../detection_engine/signals/build_rule.ts    |  72 ++++++-
 .../signals/build_signal.test.ts              |  90 +++++---
 .../detection_engine/signals/build_signal.ts  |  10 +-
 .../detection_engine/signals/get_filter.ts    |   4 +-
 .../signals/signal_params_schema.mock.ts      |   1 +
 .../signals/signal_params_schema.ts           |   1 +
 .../signals/signal_rule_alert_type.ts         |  60 +++++-
 .../signals/single_bulk_create.ts             |  62 +++++-
 .../lib/detection_engine/signals/types.ts     |  19 +-
 .../detection_engine/signals/utils.test.ts    |  53 +++++
 .../lib/detection_engine/signals/utils.ts     |  57 +++++
 .../server/lib/detection_engine/types.ts      |   2 +
 .../security_solution/server/lib/types.ts     |  55 +++--
 51 files changed, 1337 insertions(+), 108 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json

diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
index 72ef230a42342c..0224caafb41a84 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { getQueryFilter, buildExceptionFilter } from './get_query_filter';
+import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
 import { Filter, EsQueryConfig } from 'src/plugins/data/public';
 import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
 
@@ -1085,4 +1085,138 @@ describe('get_filter', () => {
       });
     });
   });
+
+  describe('buildEqlSearchRequest', () => {
+    test('should build a basic request with time range', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        undefined,
+        [],
+        undefined
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              '@timestamp': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+          },
+        },
+      });
+    });
+
+    test('should build a request with timestamp and event category overrides', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        'event.ingested',
+        [],
+        'event.other_category'
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        event_category_field: 'event.other_category',
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+          },
+        },
+      });
+    });
+
+    test('should build a request with exceptions', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        undefined,
+        [getExceptionListItemSchemaMock()],
+        undefined
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              '@timestamp': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+            bool: {
+              must_not: {
+                bool: {
+                  should: [
+                    {
+                      bool: {
+                        filter: [
+                          {
+                            nested: {
+                              path: 'some.parentField',
+                              query: {
+                                bool: {
+                                  minimum_should_match: 1,
+                                  should: [
+                                    {
+                                      match_phrase: {
+                                        'some.parentField.nested.field': 'some value',
+                                      },
+                                    },
+                                  ],
+                                },
+                              },
+                              score_mode: 'none',
+                            },
+                          },
+                          {
+                            bool: {
+                              minimum_should_match: 1,
+                              should: [
+                                {
+                                  match_phrase: {
+                                    'some.not.nested.field': 'some value',
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        },
+      });
+    });
+  });
 });
diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
index 466a004c14c660..05c706164ab44c 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
@@ -17,7 +17,12 @@ import {
   CreateExceptionListItemSchema,
 } from '../../../lists/common/schemas';
 import { buildExceptionListQueries } from './build_exceptions_query';
-import { Query as QueryString, Language, Index } from './schemas/common/schemas';
+import {
+  Query as QueryString,
+  Language,
+  Index,
+  TimestampOverrideOrUndefined,
+} from './schemas/common/schemas';
 
 export const getQueryFilter = (
   query: QueryString,
@@ -67,6 +72,78 @@ export const getQueryFilter = (
   return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
 };
 
+interface EqlSearchRequest {
+  method: string;
+  path: string;
+  body: object;
+  event_category_field?: string;
+}
+
+export const buildEqlSearchRequest = (
+  query: string,
+  index: string[],
+  from: string,
+  to: string,
+  size: number,
+  timestampOverride: TimestampOverrideOrUndefined,
+  exceptionLists: ExceptionListItemSchema[],
+  eventCategoryOverride: string | undefined
+): EqlSearchRequest => {
+  const timestamp = timestampOverride ?? '@timestamp';
+  const indexPattern: IIndexPattern = {
+    fields: [],
+    title: index.join(),
+  };
+  const config: EsQueryConfig = {
+    allowLeadingWildcards: true,
+    queryStringOptions: { analyze_wildcard: true },
+    ignoreFilterIfFieldNotInIndex: false,
+    dateFormatTZ: 'Zulu',
+  };
+  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
+  let exceptionFilter: Filter | undefined;
+  if (exceptionQueries.length > 0) {
+    // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
+    // allowing us to make 1024-item chunks of exception list items.
+    // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
+    // very conservative value.
+    exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
+  }
+  const indexString = index.join();
+  const baseRequest = {
+    method: 'POST',
+    path: `/${indexString}/_eql/search?allow_no_indices=true`,
+    body: {
+      size,
+      query,
+      filter: {
+        range: {
+          [timestamp]: {
+            gte: from,
+            lte: to,
+          },
+        },
+        bool:
+          exceptionFilter !== undefined
+            ? {
+                must_not: {
+                  bool: exceptionFilter?.query.bool,
+                },
+              }
+            : undefined,
+      },
+    },
+  };
+  if (eventCategoryOverride) {
+    return {
+      ...baseRequest,
+      event_category_field: eventCategoryOverride,
+    };
+  } else {
+    return baseRequest;
+  }
+};
+
 export const buildExceptionFilter = (
   exceptionQueries: Query[],
   indexPattern: IIndexPattern,
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
index 5fbba84467ecf4..e8d7f409de20a2 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
@@ -40,6 +40,12 @@ export type Enabled = t.TypeOf<typeof enabled>;
 export const enabledOrUndefined = t.union([enabled, t.undefined]);
 export type EnabledOrUndefined = t.TypeOf<typeof enabledOrUndefined>;
 
+export const event_category_override = t.string;
+export type EventCategoryOverride = t.TypeOf<typeof event_category_override>;
+
+export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]);
+export type EventCategoryOverrideOrUndefined = t.TypeOf<typeof eventCategoryOverrideOrUndefined>;
+
 export const false_positives = t.array(t.string);
 export type FalsePositives = t.TypeOf<typeof false_positives>;
 
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts
index 69538f025d95dd..3f338c57dd930c 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts
@@ -44,6 +44,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -96,6 +97,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanFalse, // defaults to false if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts
index c024ba1c48f8d8..2489210a26c8f3 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts
@@ -45,6 +45,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -88,6 +89,7 @@ export const createRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts
index b63d70783b7b52..a411b3d439a1f6 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts
@@ -51,6 +51,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -107,6 +108,7 @@ export const importRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts
index a674ac86af87bd..40e79d96a9e6b9 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts
@@ -46,6 +46,7 @@ import {
   timestamp_override,
   risk_score_mapping,
   severity_mapping,
+  event_category_override,
 } from '../common/schemas';
 import { listArrayOrUndefined } from '../types/lists';
 
@@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact(
     actions,
     anomaly_threshold,
     enabled,
+    event_category_override,
     false_positives,
     filters,
     from,
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts
index 1299dada065e15..8a13dd2f4e9087 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts
@@ -47,6 +47,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 
 import {
@@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override,
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
index a462b297d37f84..aaa246c82d9d77 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
@@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
   severity: 'high',
   severity_mapping: [],
   updated_by: 'elastic_kibana',
-  tags: [],
+  tags: ['some fake tag 1', 'some fake tag 2'],
   to: 'now',
   type: 'query',
   threat: [],
@@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
   status_date: '2020-02-22T16:47:50.047Z',
   last_success_at: '2020-02-22T16:47:50.047Z',
   last_success_message: 'succeeded',
-  output_index: '.siem-signals-hassanabad-frank-default',
+  output_index: '.siem-signals-default',
   max_signals: 100,
   risk_score: 55,
   risk_score_mapping: [],
@@ -110,3 +110,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R
     ],
   };
 };
+
+export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => {
+  return {
+    ...getRulesSchemaMock(anchorDate),
+    language: 'eql',
+    type: 'eql',
+    query: 'process where true',
+  };
+};
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts
index 3a47d4af6ac145..c5bad3c55066b5 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts
@@ -18,6 +18,7 @@ import {
   addTimelineTitle,
   addMlFields,
   addThreatMatchFields,
+  addEqlFields,
 } from './rules_schema';
 import { exactCheck } from '../../../exact_check';
 import { foldLeftRight, getPaths } from '../../../test_utils';
@@ -26,6 +27,7 @@ import {
   getRulesSchemaMock,
   getRulesMlSchemaMock,
   getThreatMatchingSchemaMock,
+  getRulesEqlSchemaMock,
 } from './rules_schema.mocks';
 import { ListArray } from '../types/lists';
 
@@ -628,6 +630,19 @@ describe('rules_schema', () => {
       ]);
       expect(message.schema).toEqual({});
     });
+
+    test('it validates an eql rule response', () => {
+      const payload = getRulesEqlSchemaMock();
+
+      const dependents = getDependents(payload);
+      const decoded = dependents.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = getRulesEqlSchemaMock();
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
   });
 
   describe('addSavedId', () => {
@@ -668,11 +683,6 @@ describe('rules_schema', () => {
       expect(fields.length).toEqual(2);
     });
 
-    test('should return two fields for a rule of type "eql"', () => {
-      const fields = addQueryFields({ type: 'eql' });
-      expect(fields.length).toEqual(2);
-    });
-
     test('should return two fields for a rule of type "threshold"', () => {
       const fields = addQueryFields({ type: 'threshold' });
       expect(fields.length).toEqual(2);
@@ -757,4 +767,17 @@ describe('rules_schema', () => {
       expect(fields.length).toEqual(5);
     });
   });
+
+  describe('addEqlFields', () => {
+    test('should return empty array if type is not "eql"', () => {
+      const fields = addEqlFields({ type: 'query' });
+      const expected: t.Mixed[] = [];
+      expect(fields).toEqual(expected);
+    });
+
+    test('should return 3 fields for a rule of type "eql"', () => {
+      const fields = addEqlFields({ type: 'eql' });
+      expect(fields.length).toEqual(3);
+    });
+  });
 });
diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts
index 1c2254f9f8f099..908425a7496d07 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts
@@ -16,6 +16,7 @@ import {
   anomaly_threshold,
   description,
   enabled,
+  event_category_override,
   false_positives,
   from,
   id,
@@ -121,6 +122,9 @@ export const dependentRulesSchema = t.partial({
   language,
   query,
 
+  // eql only fields
+  event_category_override,
+
   // when type = saved_query, saved_id is required
   saved_id,
 
@@ -219,9 +223,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi
 };
 
 export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => {
-  if (
-    ['eql', 'query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type)
-  ) {
+  if (['query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type)) {
     return [
       t.exact(t.type({ query: dependentRulesSchema.props.query })),
       t.exact(t.type({ language: dependentRulesSchema.props.language })),
@@ -255,6 +257,20 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.
   }
 };
 
+export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => {
+  if (typeAndTimelineOnly.type === 'eql') {
+    return [
+      t.exact(
+        t.partial({ event_category_override: dependentRulesSchema.props.event_category_override })
+      ),
+      t.exact(t.type({ query: dependentRulesSchema.props.query })),
+      t.exact(t.type({ language: dependentRulesSchema.props.language })),
+    ];
+  } else {
+    return [];
+  }
+};
+
 export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => {
   if (typeAndTimelineOnly.type === 'threat_match') {
     return [
@@ -278,6 +294,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed
     ...addQueryFields(typeAndTimelineOnly),
     ...addMlFields(typeAndTimelineOnly),
     ...addThresholdFields(typeAndTimelineOnly),
+    ...addEqlFields(typeAndTimelineOnly),
     ...addThreatMatchFields(typeAndTimelineOnly),
   ];
 
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts
index fb01f922555168..5d9cfb4bb44928 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -348,6 +348,7 @@ export const getResult = (): RuleAlertType => ({
     description: 'Detecting root and admin users',
     ruleId: 'rule-1',
     index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+    eventCategoryOverride: undefined,
     falsePositives: [],
     from: 'now-6m',
     immutable: false,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
index cfce0199100714..7255325358baf6 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
@@ -60,6 +60,16 @@
               }
             }
           },
+          "group": {
+            "properties": {
+              "id": {
+                "type": "keyword"
+              },
+              "index": {
+                "type": "integer"
+              }
+            }
+          },
           "rule": {
             "properties": {
               "id": {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
index dd887233c36a31..067a4352e10809 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
@@ -69,6 +69,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
               building_block_type: buildingBlockType,
               description,
               enabled,
+              event_category_override: eventCategoryOverride,
               false_positives: falsePositives,
               from,
               query: queryOrUndefined,
@@ -153,6 +154,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
                 buildingBlockType,
                 description,
                 enabled,
+                eventCategoryOverride,
                 falsePositives,
                 from,
                 immutable: false,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts
index 26ab89ad8ea7cd..54df87ca17787d 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { isEqlRule } from '../../../../../common/detection_engine/utils';
 import { createRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/create_rules_type_dependents';
 import { RuleAlertAction } from '../../../../../common/detection_engine/types';
 import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
@@ -53,6 +54,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void
         building_block_type: buildingBlockType,
         description,
         enabled,
+        event_category_override: eventCategoryOverride,
         false_positives: falsePositives,
         from,
         query: queryOrUndefined,
@@ -94,7 +96,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void
         const query = !isMlRule(type) && queryOrUndefined == null ? '' : queryOrUndefined;
 
         const language =
-          !isMlRule(type) && languageOrUndefined == null ? 'kuery' : languageOrUndefined;
+          !isMlRule(type) && !isEqlRule(type) && languageOrUndefined == null
+            ? 'kuery'
+            : languageOrUndefined;
 
         // TODO: Fix these either with an is conversion or by better typing them within io-ts
         const actions: RuleAlertAction[] = actionsRest as RuleAlertAction[];
@@ -138,6 +142,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void
           buildingBlockType,
           description,
           enabled,
+          eventCategoryOverride,
           falsePositives,
           from,
           immutable: false,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts
index 0f5d0304f5ca02..4dbca5df0041cf 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts
@@ -135,6 +135,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP
                   building_block_type: buildingBlockType,
                   description,
                   enabled,
+                  event_category_override: eventCategoryOverride,
                   false_positives: falsePositives,
                   from,
                   immutable,
@@ -194,6 +195,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP
                       buildingBlockType,
                       description,
                       enabled,
+                      eventCategoryOverride,
                       falsePositives,
                       from,
                       immutable,
@@ -242,6 +244,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP
                       savedObjectsClient,
                       description,
                       enabled,
+                      eventCategoryOverride,
                       falsePositives,
                       from,
                       query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts
index 5099cf5de958fe..39bbe9ee686a49 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts
@@ -59,6 +59,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
             building_block_type: buildingBlockType,
             description,
             enabled,
+            event_category_override: eventCategoryOverride,
             false_positives: falsePositives,
             from,
             query,
@@ -119,6 +120,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
               buildingBlockType,
               description,
               enabled,
+              eventCategoryOverride,
               falsePositives,
               from,
               query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts
index 3b3efd2ed166dc..879bd8d5b8a1d1 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts
@@ -50,6 +50,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => {
         building_block_type: buildingBlockType,
         description,
         enabled,
+        event_category_override: eventCategoryOverride,
         false_positives: falsePositives,
         from,
         query,
@@ -117,6 +118,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => {
           buildingBlockType,
           description,
           enabled,
+          eventCategoryOverride,
           falsePositives,
           from,
           query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
index 0e414e130849a5..4df0773f86317a 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts
@@ -62,6 +62,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
             building_block_type: buildingBlockType,
             description,
             enabled,
+            event_category_override: eventCategoryOverride,
             false_positives: falsePositives,
             from,
             query: queryOrUndefined,
@@ -127,6 +128,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
               buildingBlockType,
               description,
               enabled,
+              eventCategoryOverride,
               falsePositives,
               from,
               query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts
index 553d084b626337..ef698db008d804 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts
@@ -52,6 +52,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => {
         building_block_type: buildingBlockType,
         description,
         enabled,
+        event_category_override: eventCategoryOverride,
         false_positives: falsePositives,
         from,
         query: queryOrUndefined,
@@ -117,6 +118,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => {
           buildingBlockType,
           description,
           enabled,
+          eventCategoryOverride,
           falsePositives,
           from,
           query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts
index 556ea209152e6c..c75b32b614e078 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts
@@ -114,6 +114,7 @@ export const transformAlertToRule = (
     description: alert.params.description,
     enabled: alert.enabled,
     anomaly_threshold: alert.params.anomalyThreshold,
+    event_category_override: alert.params.eventCategoryOverride,
     false_positives: alert.params.falsePositives,
     filters: alert.params.filters,
     from: alert.params.from,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts
index 95067e57868d14..a6034f3d7b7b37 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts
@@ -14,6 +14,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({
   anomalyThreshold: undefined,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: 'user.name: root or user.name: admin',
@@ -61,6 +62,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({
   anomalyThreshold: 55,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: undefined,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts
index 9ed94cd7bff2e6..3a311d03e3c89f 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts
@@ -17,6 +17,7 @@ export const createRules = async ({
   buildingBlockType,
   description,
   enabled,
+  eventCategoryOverride,
   falsePositives,
   from,
   query,
@@ -69,6 +70,7 @@ export const createRules = async ({
         description,
         ruleId,
         index,
+        eventCategoryOverride,
         falsePositives,
         from,
         immutable,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts
index 59e14dcffc3c01..38adc03c00d502 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts
@@ -22,6 +22,7 @@ export const installPrepackagedRules = (
       building_block_type: buildingBlockType,
       description,
       enabled,
+      event_category_override: eventCategoryOverride,
       false_positives: falsePositives,
       from,
       query,
@@ -70,6 +71,7 @@ export const installPrepackagedRules = (
         buildingBlockType,
         description,
         enabled,
+        eventCategoryOverride,
         falsePositives,
         from,
         immutable: true, // At the moment we force all prepackaged rules to be immutable
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts
index cfb40056eb85d1..aeb136a969aa18 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts
@@ -120,6 +120,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({
   anomalyThreshold: undefined,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: 'user.name: root or user.name: admin',
@@ -163,6 +164,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({
   anomalyThreshold: 55,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: undefined,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts
index e0814647b4c39a..852ff06bdc736c 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts
@@ -18,6 +18,7 @@ export const patchRules = async ({
   buildingBlockType,
   savedObjectsClient,
   description,
+  eventCategoryOverride,
   falsePositives,
   enabled,
   query,
@@ -62,6 +63,7 @@ export const patchRules = async ({
     author,
     buildingBlockType,
     description,
+    eventCategoryOverride,
     falsePositives,
     query,
     language,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts
index 6b851351f27f25..d688e1b338e21b 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts
@@ -84,6 +84,7 @@ import {
   TimestampOverrideOrUndefined,
   BuildingBlockTypeOrUndefined,
   RuleNameOverrideOrUndefined,
+  EventCategoryOverrideOrUndefined,
 } from '../../../../common/detection_engine/schemas/common/schemas';
 import {
   ThreatIndexOrUndefined,
@@ -187,6 +188,7 @@ export interface CreateRulesOptions {
   buildingBlockType: BuildingBlockTypeOrUndefined;
   description: Description;
   enabled: Enabled;
+  eventCategoryOverride: EventCategoryOverrideOrUndefined;
   falsePositives: FalsePositives;
   from: From;
   query: QueryOrUndefined;
@@ -236,6 +238,7 @@ export interface UpdateRulesOptions {
   buildingBlockType: BuildingBlockTypeOrUndefined;
   description: Description;
   enabled: Enabled;
+  eventCategoryOverride: EventCategoryOverrideOrUndefined;
   falsePositives: FalsePositives;
   from: From;
   query: QueryOrUndefined;
@@ -279,6 +282,7 @@ export interface PatchRulesOptions {
   buildingBlockType: BuildingBlockTypeOrUndefined;
   description: DescriptionOrUndefined;
   enabled: EnabledOrUndefined;
+  eventCategoryOverride: EventCategoryOverrideOrUndefined;
   falsePositives: FalsePositivesOrUndefined;
   from: FromOrUndefined;
   query: QueryOrUndefined;
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts
index bf97784e8d917e..01a481ed7b2d9b 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts
@@ -23,6 +23,7 @@ export const updatePrepackagedRules = async (
         author,
         building_block_type: buildingBlockType,
         description,
+        event_category_override: eventCategoryOverride,
         false_positives: falsePositives,
         from,
         query,
@@ -69,6 +70,7 @@ export const updatePrepackagedRules = async (
         author,
         buildingBlockType,
         description,
+        eventCategoryOverride,
         falsePositives,
         from,
         query,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts
index 650b59fb85bc03..8cdc904a861c7a 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts
@@ -17,6 +17,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({
   anomalyThreshold: undefined,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: 'user.name: root or user.name: admin',
@@ -61,6 +62,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({
   anomalyThreshold: 55,
   description: 'some description',
   enabled: true,
+  eventCategoryOverride: undefined,
   falsePositives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
   query: undefined,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts
index 494a4e221d8629..08df785884b76b 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts
@@ -18,6 +18,7 @@ export const updateRules = async ({
   buildingBlockType,
   savedObjectsClient,
   description,
+  eventCategoryOverride,
   falsePositives,
   enabled,
   query,
@@ -64,6 +65,7 @@ export const updateRules = async ({
     author,
     buildingBlockType,
     description,
+    eventCategoryOverride,
     falsePositives,
     query,
     language,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts
index 17505a44782618..227f574bc4e4b4 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts
@@ -31,6 +31,7 @@ describe('utils', () => {
           author: [],
           buildingBlockType: undefined,
           description: 'some description change',
+          eventCategoryOverride: undefined,
           falsePositives: undefined,
           query: undefined,
           language: undefined,
@@ -73,6 +74,7 @@ describe('utils', () => {
           author: [],
           buildingBlockType: undefined,
           description: 'some description change',
+          eventCategoryOverride: undefined,
           falsePositives: undefined,
           query: undefined,
           language: undefined,
@@ -115,6 +117,7 @@ describe('utils', () => {
           author: [],
           buildingBlockType: undefined,
           description: 'some description change',
+          eventCategoryOverride: undefined,
           falsePositives: undefined,
           query: undefined,
           language: undefined,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts
index 49c02f92ff3361..d9f953f2803a61 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts
@@ -39,6 +39,7 @@ import {
   RuleNameOverrideOrUndefined,
   SeverityMappingOrUndefined,
   TimestampOverrideOrUndefined,
+  EventCategoryOverrideOrUndefined,
 } from '../../../../common/detection_engine/schemas/common/schemas';
 import { PartialFilter } from '../types';
 import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types';
@@ -60,6 +61,7 @@ export interface UpdateProperties {
   author: AuthorOrUndefined;
   buildingBlockType: BuildingBlockTypeOrUndefined;
   description: DescriptionOrUndefined;
+  eventCategoryOverride: EventCategoryOverrideOrUndefined;
   falsePositives: FalsePositivesOrUndefined;
   from: FromOrUndefined;
   query: QueryOrUndefined;
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json
new file mode 100644
index 00000000000000..598f2182002c1d
--- /dev/null
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json
@@ -0,0 +1,68 @@
+{
+  "name": "EQL query rule",
+  "description": "Rule with an eql query",
+  "false_positives": [
+    "https://www.example.com/some-article-about-a-false-positive",
+    "some text string about why another condition could be a false positive"
+  ],
+  "rule_id": "rule-id-eql",
+  "enabled": false,
+  "index": [".ds-logs-endpoint.events.process-default-000001"],
+  "interval": "30s",
+  "query": "sequence [process where process.name = \"mimikatz.exe\"] [process where process.name = \"explorer.exe\"]",
+  "output_index": ".siem-signals-default",
+  "meta": {
+    "anything_you_want_ui_related_or_otherwise": {
+      "as_deep_structured_as_you_need": {
+        "any_data_type": {}
+      }
+    }
+  },
+  "risk_score": 1,
+  "max_signals": 100,
+  "tags": ["tag 1", "tag 2", "any tag you want"],
+  "to": "now",
+  "from": "now-300m",
+  "severity": "high",
+  "type": "eql",
+  "threat": [
+    {
+      "framework": "MITRE ATT&CK",
+      "tactic": {
+        "id": "TA0040",
+        "name": "impact",
+        "reference": "https://attack.mitre.org/tactics/TA0040/"
+      },
+      "technique": [
+        {
+          "id": "T1499",
+          "name": "endpoint denial of service",
+          "reference": "https://attack.mitre.org/techniques/T1499/"
+        }
+      ]
+    },
+    {
+      "framework": "Some other Framework you want",
+      "tactic": {
+        "id": "some-other-id",
+        "name": "Some other name",
+        "reference": "https://example.com"
+      },
+      "technique": [
+        {
+          "id": "some-other-id",
+          "name": "some other technique name",
+          "reference": "https://example.com"
+        }
+      ]
+    }
+  ],
+  "references": [
+    "http://www.example.com/some-article-about-attack",
+    "Some plain text string here explaining why this is a valid thing to look out for"
+  ],
+  "timeline_id": "timeline_id",
+  "timeline_title": "timeline_title",
+  "note": "# note markdown",
+  "version": 1
+}
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
index 9ee8c5cf298a16..b37bc7d0fab69c 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
@@ -4,7 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SignalSourceHit, SignalSearchResponse, BulkResponse, BulkItem } from '../types';
+import {
+  SignalSourceHit,
+  SignalSearchResponse,
+  BulkResponse,
+  BulkItem,
+  RuleAlertAttributes,
+  SignalHit,
+} from '../types';
 import {
   Logger,
   SavedObject,
@@ -24,6 +31,7 @@ export const sampleRuleAlertParams = (
   buildingBlockType: 'default',
   ruleId: 'rule-1',
   description: 'Detecting root and admin users',
+  eventCategoryOverride: undefined,
   falsePositives: [],
   immutable: false,
   index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
@@ -60,6 +68,30 @@ export const sampleRuleAlertParams = (
   exceptionsList: getListArrayMock(),
 });
 
+export const sampleRuleSO = (): SavedObject<RuleAlertAttributes> => {
+  return {
+    id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+    type: 'alert',
+    version: '1',
+    updated_at: '2020-03-27T22:55:59.577Z',
+    attributes: {
+      actions: [],
+      enabled: true,
+      name: 'rule-name',
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      createdBy: 'sample user',
+      createdAt: '2020-03-27T22:55:59.577Z',
+      updatedBy: 'sample user',
+      schedule: {
+        interval: '5m',
+      },
+      throttle: 'no_actions',
+      params: sampleRuleAlertParams(),
+    },
+    references: [],
+  };
+};
+
 export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({
   _index: 'myFakeSignalIndex',
   _type: 'doc',
@@ -189,6 +221,80 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => {
   };
 };
 
+export const sampleSignalHit = (): SignalHit => ({
+  '@timestamp': '2020-04-20T21:27:45+0000',
+  event: {
+    kind: 'signal',
+  },
+  signal: {
+    parents: [
+      {
+        id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+      {
+        id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+    ],
+    ancestors: [
+      {
+        id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+      {
+        id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+    ],
+    status: 'open',
+    rule: {
+      author: [],
+      id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+      created_at: '2020-04-20T21:27:45+0000',
+      updated_at: '2020-04-20T21:27:45+0000',
+      created_by: 'elastic',
+      description: 'some description',
+      enabled: true,
+      false_positives: ['false positive 1', 'false positive 2'],
+      from: 'now-6m',
+      immutable: false,
+      name: 'Query with a rule id',
+      query: 'user.name: root or user.name: admin',
+      references: ['test 1', 'test 2'],
+      severity: 'high',
+      severity_mapping: [],
+      updated_by: 'elastic_kibana',
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      to: 'now',
+      type: 'query',
+      threat: [],
+      version: 1,
+      status: 'succeeded',
+      status_date: '2020-02-22T16:47:50.047Z',
+      last_success_at: '2020-02-22T16:47:50.047Z',
+      last_success_message: 'succeeded',
+      output_index: '.siem-signals-default',
+      max_signals: 100,
+      risk_score: 55,
+      risk_score_mapping: [],
+      language: 'kuery',
+      rule_id: 'query-rule-id',
+      interval: '5m',
+      exceptions_list: getListArrayMock(),
+    },
+    depth: 1,
+  },
+});
+
 export const sampleBulkCreateDuplicateResult = {
   took: 60,
   errors: true,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
index 967dc5331e46b1..f45a408cd32b8e 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
@@ -9,8 +9,10 @@ import {
   sampleDocNoSortId,
   sampleRuleGuid,
   sampleIdGuid,
+  sampleDocWithAncestors,
+  sampleRuleSO,
 } from './__mocks__/es_results';
-import { buildBulkBody } from './build_bulk_body';
+import { buildBulkBody, buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body';
 import { SignalHit } from './types';
 import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
 
@@ -433,3 +435,200 @@ describe('buildBulkBody', () => {
     expect(fakeSignalSourceHit).toEqual(expected);
   });
 });
+
+describe('buildSignalFromSequence', () => {
+  test('builds a basic signal from a sequence of building blocks', () => {
+    const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]];
+    const ruleSO = sampleRuleSO();
+    const signal = buildSignalFromSequence(blocks, ruleSO);
+    // Timestamp will potentially always be different so remove it for the test
+    // @ts-expect-error
+    delete signal['@timestamp'];
+    const expected: Omit<SignalHit, '@timestamp'> = {
+      event: {
+        kind: 'signal',
+      },
+      signal: {
+        parents: [
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+        ],
+        ancestors: [
+          {
+            id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+            type: 'event',
+            index: 'myFakeSignalIndex',
+            depth: 0,
+          },
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+          {
+            id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+            type: 'event',
+            index: 'myFakeSignalIndex',
+            depth: 0,
+          },
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+        ],
+        status: 'open',
+        rule: {
+          actions: [],
+          author: ['Elastic'],
+          building_block_type: 'default',
+          id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+          rule_id: 'rule-1',
+          false_positives: [],
+          max_signals: 10000,
+          risk_score: 50,
+          risk_score_mapping: [],
+          output_index: '.siem-signals',
+          description: 'Detecting root and admin users',
+          from: 'now-6m',
+          immutable: false,
+          index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+          interval: '5m',
+          language: 'kuery',
+          license: 'Elastic License',
+          name: 'rule-name',
+          query: 'user.name: root or user.name: admin',
+          references: ['http://google.com'],
+          severity: 'high',
+          severity_mapping: [],
+          tags: ['some fake tag 1', 'some fake tag 2'],
+          threat: [],
+          type: 'query',
+          to: 'now',
+          note: '',
+          enabled: true,
+          created_by: 'sample user',
+          updated_by: 'sample user',
+          version: 1,
+          updated_at: ruleSO.updated_at ?? '',
+          created_at: ruleSO.attributes.createdAt,
+          throttle: 'no_actions',
+          exceptions_list: getListArrayMock(),
+        },
+        depth: 2,
+        group: {
+          id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea',
+        },
+      },
+    };
+    expect(signal).toEqual(expected);
+  });
+});
+
+describe('buildSignalFromEvent', () => {
+  test('builds a basic signal from a single event', () => {
+    const ancestor = sampleDocWithAncestors().hits.hits[0];
+    delete ancestor._source.source;
+    const ruleSO = sampleRuleSO();
+    const signal = buildSignalFromEvent(ancestor, ruleSO);
+    // Timestamp will potentially always be different so remove it for the test
+    // @ts-expect-error
+    delete signal['@timestamp'];
+    const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
+      someKey: 'someValue',
+      event: {
+        kind: 'signal',
+      },
+      signal: {
+        original_time: '2020-04-20T21:27:45+0000',
+        parent: {
+          id: sampleIdGuid,
+          rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+          type: 'signal',
+          index: 'myFakeSignalIndex',
+          depth: 1,
+        },
+        parents: [
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+        ],
+        ancestors: [
+          {
+            id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+            type: 'event',
+            index: 'myFakeSignalIndex',
+            depth: 0,
+          },
+          {
+            id: sampleIdGuid,
+            rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+            type: 'signal',
+            index: 'myFakeSignalIndex',
+            depth: 1,
+          },
+        ],
+        status: 'open',
+        rule: {
+          actions: [],
+          author: ['Elastic'],
+          building_block_type: 'default',
+          id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+          rule_id: 'rule-1',
+          false_positives: [],
+          max_signals: 10000,
+          risk_score: 50,
+          risk_score_mapping: [],
+          output_index: '.siem-signals',
+          description: 'Detecting root and admin users',
+          from: 'now-6m',
+          immutable: false,
+          index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+          interval: '5m',
+          language: 'kuery',
+          license: 'Elastic License',
+          name: 'rule-name',
+          query: 'user.name: root or user.name: admin',
+          references: ['http://google.com'],
+          severity: 'high',
+          severity_mapping: [],
+          tags: ['some fake tag 1', 'some fake tag 2'],
+          threat: [],
+          type: 'query',
+          to: 'now',
+          note: '',
+          enabled: true,
+          created_by: 'sample user',
+          updated_by: 'sample user',
+          version: 1,
+          updated_at: ruleSO.updated_at ?? '',
+          created_at: ruleSO.attributes.createdAt,
+          throttle: 'no_actions',
+          exceptions_list: getListArrayMock(),
+        },
+        depth: 2,
+      },
+    };
+    expect(signal).toEqual(expected);
+  });
+});
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
index 7be97e46f91f24..01a6b0e7aefadb 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
@@ -4,12 +4,22 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SignalSourceHit, SignalHit, Signal } from './types';
-import { buildRule } from './build_rule';
+import { SavedObject } from 'src/core/types';
+import {
+  SignalSourceHit,
+  SignalHit,
+  Signal,
+  RuleAlertAttributes,
+  BaseSignalHit,
+  SignalSource,
+} from './types';
+import { buildRule, buildRuleWithoutOverrides } from './build_rule';
 import { additionalSignalFields, buildSignal } from './build_signal';
 import { buildEventTypeSignal } from './build_event_type_signal';
 import { RuleAlertAction } from '../../../../common/detection_engine/types';
 import { RuleTypeParams } from '../types';
+import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils';
+import { EqlSequence } from '../../types';
 
 interface BuildBulkBodyParams {
   doc: SignalSourceHit;
@@ -71,3 +81,86 @@ export const buildBulkBody = ({
   };
   return signalHit;
 };
+
+/**
+ * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed -
+ * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals
+ * share the same signal.group.id to make it easy to query them.
+ * @param sequence The raw ES documents that make up the sequence
+ * @param ruleSO SavedObject representing the rule that found the sequence
+ * @param outputIndex Index to write the resulting signals to
+ */
+export const buildSignalGroupFromSequence = (
+  sequence: EqlSequence<SignalSource>,
+  ruleSO: SavedObject<RuleAlertAttributes>,
+  outputIndex: string
+): BaseSignalHit[] => {
+  const wrappedBuildingBlocks = wrapBuildingBlocks(
+    sequence.events.map((event) => {
+      const signal = buildSignalFromEvent(event, ruleSO);
+      signal.signal.rule.building_block_type = 'default';
+      return signal;
+    }),
+    outputIndex
+  );
+
+  // Now that we have an array of building blocks for the events in the sequence,
+  // we can build the signal that links the building blocks together
+  // and also insert the group id (which is also the "shell" signal _id) in each building block
+  const sequenceSignal = wrapSignal(
+    buildSignalFromSequence(wrappedBuildingBlocks, ruleSO),
+    outputIndex
+  );
+  wrappedBuildingBlocks.forEach((block, idx) => {
+    // TODO: fix type of blocks so we don't have to check existence of _source.signal
+    if (block._source.signal) {
+      block._source.signal.group = {
+        id: sequenceSignal._id,
+        index: idx,
+      };
+    }
+  });
+  return [...wrappedBuildingBlocks, sequenceSignal];
+};
+
+export const buildSignalFromSequence = (
+  events: BaseSignalHit[],
+  ruleSO: SavedObject<RuleAlertAttributes>
+): SignalHit => {
+  const rule = buildRuleWithoutOverrides(ruleSO);
+  const signal: Signal = buildSignal(events, rule);
+  return {
+    '@timestamp': new Date().toISOString(),
+    event: {
+      kind: 'signal',
+    },
+    signal: {
+      ...signal,
+      group: {
+        // This is the same function that is used later to generate the _id for the sequence signal document,
+        // so _id should equal signal.group.id for the "shell" document
+        id: generateSignalId(signal),
+      },
+    },
+  };
+};
+
+export const buildSignalFromEvent = (
+  event: BaseSignalHit,
+  ruleSO: SavedObject<RuleAlertAttributes>
+): SignalHit => {
+  const rule = buildRuleWithoutOverrides(ruleSO);
+  const signal = {
+    ...buildSignal([event], rule),
+    ...additionalSignalFields(event),
+  };
+  const eventFields = buildEventTypeSignal(event);
+  // TODO: better naming for SignalHit - it's really a new signal to be inserted
+  const signalHit: SignalHit = {
+    ...event._source,
+    '@timestamp': new Date().toISOString(),
+    event: eventFields,
+    signal,
+  };
+  return signalHit;
+};
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts
index 59cdc020c611d4..81c9d1dedcc56c 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts
@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SignalSourceHit } from './types';
+import { BaseSignalHit } from './types';
 
-export const buildEventTypeSignal = (doc: SignalSourceHit): object => {
+export const buildEventTypeSignal = (doc: BaseSignalHit): object => {
   if (doc._source.event != null && doc._source.event instanceof Object) {
     return { ...doc._source.event, kind: 'signal' };
   } else {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts
index ba815a0b62f0d3..62e5854037d9e1 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts
@@ -4,12 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { buildRule, removeInternalTagsFromRule } from './build_rule';
-import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results';
+import { buildRule, removeInternalTagsFromRule, buildRuleWithoutOverrides } from './build_rule';
+import {
+  sampleDocNoSortId,
+  sampleRuleAlertParams,
+  sampleRuleGuid,
+  sampleRuleSO,
+} from './__mocks__/es_results';
 import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
 import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
 import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants';
-import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks';
+import { getRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks';
 
 describe('buildRule', () => {
   beforeEach(() => {
@@ -272,9 +277,11 @@ describe('buildRule', () => {
     };
     expect(rule).toEqual(expected);
   });
+});
 
+describe('removeInternalTagsFromRule', () => {
   test('it removes internal tags from a typical rule', () => {
-    const rule = getPartialRulesSchemaMock();
+    const rule = getRulesSchemaMock();
     rule.tags = [
       'some fake tag 1',
       'some fake tag 2',
@@ -282,30 +289,113 @@ describe('buildRule', () => {
       `${INTERNAL_IMMUTABLE_KEY}:true`,
     ];
     const noInternals = removeInternalTagsFromRule(rule);
-    expect(noInternals).toEqual(getPartialRulesSchemaMock());
+    expect(noInternals).toEqual(getRulesSchemaMock());
   });
 
   test('it works with an empty array', () => {
-    const rule = getPartialRulesSchemaMock();
+    const rule = getRulesSchemaMock();
     rule.tags = [];
     const noInternals = removeInternalTagsFromRule(rule);
-    const expected = getPartialRulesSchemaMock();
+    const expected = getRulesSchemaMock();
     expected.tags = [];
     expect(noInternals).toEqual(expected);
   });
 
-  test('it works if tags does not exist', () => {
-    const rule = getPartialRulesSchemaMock();
-    delete rule.tags;
-    const noInternals = removeInternalTagsFromRule(rule);
-    const expected = getPartialRulesSchemaMock();
-    delete expected.tags;
-    expect(noInternals).toEqual(expected);
-  });
-
   test('it works if tags contains normal values and no internal values', () => {
-    const rule = getPartialRulesSchemaMock();
+    const rule = getRulesSchemaMock();
     const noInternals = removeInternalTagsFromRule(rule);
     expect(noInternals).toEqual(rule);
   });
 });
+
+describe('buildRuleWithoutOverrides', () => {
+  test('builds a rule using rule SO', () => {
+    const ruleSO = sampleRuleSO();
+    const rule = buildRuleWithoutOverrides(ruleSO);
+    expect(rule).toEqual({
+      actions: [],
+      author: ['Elastic'],
+      building_block_type: 'default',
+      id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+      rule_id: 'rule-1',
+      false_positives: [],
+      max_signals: 10000,
+      risk_score: 50,
+      risk_score_mapping: [],
+      output_index: '.siem-signals',
+      description: 'Detecting root and admin users',
+      from: 'now-6m',
+      immutable: false,
+      index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      interval: '5m',
+      language: 'kuery',
+      license: 'Elastic License',
+      name: 'rule-name',
+      query: 'user.name: root or user.name: admin',
+      references: ['http://google.com'],
+      severity: 'high',
+      severity_mapping: [],
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      threat: [],
+      type: 'query',
+      to: 'now',
+      note: '',
+      enabled: true,
+      created_by: 'sample user',
+      updated_by: 'sample user',
+      version: 1,
+      updated_at: ruleSO.updated_at ?? '',
+      created_at: ruleSO.attributes.createdAt,
+      throttle: 'no_actions',
+      exceptions_list: getListArrayMock(),
+    });
+  });
+
+  test('builds a rule using rule SO and removes internal tags', () => {
+    const ruleSO = sampleRuleSO();
+    ruleSO.attributes.tags = [
+      'some fake tag 1',
+      'some fake tag 2',
+      `${INTERNAL_RULE_ID_KEY}:rule-1`,
+      `${INTERNAL_IMMUTABLE_KEY}:true`,
+    ];
+    const rule = buildRuleWithoutOverrides(ruleSO);
+    expect(rule).toEqual({
+      actions: [],
+      author: ['Elastic'],
+      building_block_type: 'default',
+      id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+      rule_id: 'rule-1',
+      false_positives: [],
+      max_signals: 10000,
+      risk_score: 50,
+      risk_score_mapping: [],
+      output_index: '.siem-signals',
+      description: 'Detecting root and admin users',
+      from: 'now-6m',
+      immutable: false,
+      index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      interval: '5m',
+      language: 'kuery',
+      license: 'Elastic License',
+      name: 'rule-name',
+      query: 'user.name: root or user.name: admin',
+      references: ['http://google.com'],
+      severity: 'high',
+      severity_mapping: [],
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      threat: [],
+      type: 'query',
+      to: 'now',
+      note: '',
+      enabled: true,
+      created_by: 'sample user',
+      updated_by: 'sample user',
+      version: 1,
+      updated_at: ruleSO.updated_at ?? '',
+      created_at: ruleSO.attributes.createdAt,
+      throttle: 'no_actions',
+      exceptions_list: getListArrayMock(),
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts
index aacf9b8be31b41..e5370735333bc7 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts
@@ -4,12 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { pickBy } from 'lodash/fp';
+import { SavedObject } from 'src/core/types';
 import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
 import { RuleAlertAction } from '../../../../common/detection_engine/types';
 import { RuleTypeParams } from '../types';
 import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping';
-import { SignalSourceHit } from './types';
+import { SignalSourceHit, RuleAlertAttributes } from './types';
 import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping';
 import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping';
 import { INTERNAL_IDENTIFIER } from '../../../../common/constants';
@@ -44,7 +44,7 @@ export const buildRule = ({
   interval,
   tags,
   throttle,
-}: BuildRuleParams): Partial<RulesSchema> => {
+}: BuildRuleParams): RulesSchema => {
   const { riskScore, riskScoreMeta } = buildRiskScoreFromMapping({
     doc,
     riskScore: ruleParams.riskScore,
@@ -65,7 +65,7 @@ export const buildRule = ({
 
   const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta };
 
-  const rule = pickBy<RulesSchema>((value: unknown) => value != null, {
+  const rule = {
     id,
     rule_id: ruleParams.ruleId ?? '(unknown rule_id)',
     actions,
@@ -111,15 +111,73 @@ export const buildRule = ({
     machine_learning_job_id: ruleParams.machineLearningJobId,
     anomaly_threshold: ruleParams.anomalyThreshold,
     threshold: ruleParams.threshold,
-  });
+  };
+  return removeInternalTagsFromRule(rule);
+};
+
+export const buildRuleWithoutOverrides = (
+  ruleSO: SavedObject<RuleAlertAttributes>
+): RulesSchema => {
+  const ruleParams = ruleSO.attributes.params;
+  const rule: RulesSchema = {
+    id: ruleSO.id,
+    rule_id: ruleParams.ruleId,
+    actions: ruleSO.attributes.actions,
+    author: ruleParams.author ?? [],
+    building_block_type: ruleParams.buildingBlockType,
+    false_positives: ruleParams.falsePositives,
+    saved_id: ruleParams.savedId,
+    timeline_id: ruleParams.timelineId,
+    timeline_title: ruleParams.timelineTitle,
+    meta: ruleParams.meta,
+    max_signals: ruleParams.maxSignals,
+    risk_score: ruleParams.riskScore,
+    risk_score_mapping: ruleParams.riskScoreMapping ?? [],
+    output_index: ruleParams.outputIndex,
+    description: ruleParams.description,
+    note: ruleParams.note,
+    from: ruleParams.from,
+    immutable: ruleParams.immutable,
+    index: ruleParams.index,
+    interval: ruleSO.attributes.schedule.interval,
+    language: ruleParams.language,
+    license: ruleParams.license,
+    name: ruleSO.attributes.name,
+    query: ruleParams.query,
+    references: ruleParams.references,
+    rule_name_override: ruleParams.ruleNameOverride,
+    severity: ruleParams.severity,
+    severity_mapping: ruleParams.severityMapping ?? [],
+    tags: ruleSO.attributes.tags,
+    type: ruleParams.type,
+    to: ruleParams.to,
+    enabled: ruleSO.attributes.enabled,
+    filters: ruleParams.filters,
+    created_by: ruleSO.attributes.createdBy,
+    updated_by: ruleSO.attributes.updatedBy,
+    threat: ruleParams.threat ?? [],
+    timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override
+    throttle: ruleSO.attributes.throttle,
+    version: ruleParams.version,
+    created_at: ruleSO.attributes.createdAt,
+    updated_at: ruleSO.updated_at ?? '',
+    exceptions_list: ruleParams.exceptionsList ?? [],
+    machine_learning_job_id: ruleParams.machineLearningJobId,
+    anomaly_threshold: ruleParams.anomalyThreshold,
+    threshold: ruleParams.threshold,
+    threat_filters: ruleParams.threatFilters,
+    threat_index: ruleParams.threatIndex,
+    threat_query: ruleParams.threatQuery,
+    threat_mapping: ruleParams.threatMapping,
+  };
   return removeInternalTagsFromRule(rule);
 };
 
-export const removeInternalTagsFromRule = (rule: Partial<RulesSchema>): Partial<RulesSchema> => {
+export const removeInternalTagsFromRule = (rule: RulesSchema): RulesSchema => {
   if (rule.tags == null) {
     return rule;
   } else {
-    const ruleWithoutInternalTags: Partial<RulesSchema> = {
+    const ruleWithoutInternalTags: RulesSchema = {
       ...rule,
       tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)),
     };
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts
index d684807a09126f..d0c451bbdf2e2b 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts
@@ -7,7 +7,11 @@
 import { sampleDocNoSortId } from './__mocks__/es_results';
 import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal';
 import { Signal, Ancestor } from './types';
-import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks';
+import {
+  getRulesSchemaMock,
+  ANCHOR_DATE,
+} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks';
+import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
 
 describe('buildSignal', () => {
   beforeEach(() => {
@@ -17,7 +21,7 @@ describe('buildSignal', () => {
   test('it builds a signal as expected without original_event if event does not exist', () => {
     const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
     delete doc._source.event;
-    const rule = getPartialRulesSchemaMock();
+    const rule = getRulesSchemaMock();
     const signal = {
       ...buildSignal([doc], rule),
       ...additionalSignalFields(doc),
@@ -48,31 +52,39 @@ describe('buildSignal', () => {
       original_time: '2020-04-20T21:27:45+0000',
       status: 'open',
       rule: {
+        author: [],
+        id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+        created_at: new Date(ANCHOR_DATE).toISOString(),
+        updated_at: new Date(ANCHOR_DATE).toISOString(),
         created_by: 'elastic',
-        description: 'Detecting root and admin users',
+        description: 'some description',
         enabled: true,
-        false_positives: [],
+        false_positives: ['false positive 1', 'false positive 2'],
         from: 'now-6m',
-        id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
         immutable: false,
-        index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
-        interval: '5m',
-        risk_score: 50,
-        rule_id: 'rule-1',
-        language: 'kuery',
-        max_signals: 100,
-        name: 'Detect Root/Admin Users',
-        output_index: '.siem-signals',
+        name: 'Query with a rule id',
         query: 'user.name: root or user.name: admin',
-        references: ['http://www.example.com', 'https://ww.example.com'],
+        references: ['test 1', 'test 2'],
         severity: 'high',
-        updated_by: 'elastic',
+        severity_mapping: [],
+        updated_by: 'elastic_kibana',
         tags: ['some fake tag 1', 'some fake tag 2'],
         to: 'now',
         type: 'query',
-        note: '',
-        updated_at: signal.rule.updated_at,
-        created_at: signal.rule.created_at,
+        threat: [],
+        version: 1,
+        status: 'succeeded',
+        status_date: '2020-02-22T16:47:50.047Z',
+        last_success_at: '2020-02-22T16:47:50.047Z',
+        last_success_message: 'succeeded',
+        output_index: '.siem-signals-default',
+        max_signals: 100,
+        risk_score: 55,
+        risk_score_mapping: [],
+        language: 'kuery',
+        rule_id: 'query-rule-id',
+        interval: '5m',
+        exceptions_list: getListArrayMock(),
       },
       depth: 1,
     };
@@ -87,7 +99,7 @@ describe('buildSignal', () => {
       kind: 'event',
       module: 'system',
     };
-    const rule = getPartialRulesSchemaMock();
+    const rule = getRulesSchemaMock();
     const signal = {
       ...buildSignal([doc], rule),
       ...additionalSignalFields(doc),
@@ -124,31 +136,39 @@ describe('buildSignal', () => {
       },
       status: 'open',
       rule: {
+        author: [],
+        id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+        created_at: new Date(ANCHOR_DATE).toISOString(),
+        updated_at: new Date(ANCHOR_DATE).toISOString(),
         created_by: 'elastic',
-        description: 'Detecting root and admin users',
+        description: 'some description',
         enabled: true,
-        false_positives: [],
+        false_positives: ['false positive 1', 'false positive 2'],
         from: 'now-6m',
-        id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
         immutable: false,
-        index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
-        interval: '5m',
-        risk_score: 50,
-        rule_id: 'rule-1',
-        language: 'kuery',
-        max_signals: 100,
-        name: 'Detect Root/Admin Users',
-        output_index: '.siem-signals',
+        name: 'Query with a rule id',
         query: 'user.name: root or user.name: admin',
-        references: ['http://www.example.com', 'https://ww.example.com'],
+        references: ['test 1', 'test 2'],
         severity: 'high',
-        updated_by: 'elastic',
+        severity_mapping: [],
+        updated_by: 'elastic_kibana',
         tags: ['some fake tag 1', 'some fake tag 2'],
         to: 'now',
         type: 'query',
-        note: '',
-        updated_at: signal.rule.updated_at,
-        created_at: signal.rule.created_at,
+        threat: [],
+        version: 1,
+        status: 'succeeded',
+        status_date: '2020-02-22T16:47:50.047Z',
+        last_success_at: '2020-02-22T16:47:50.047Z',
+        last_success_message: 'succeeded',
+        output_index: '.siem-signals-default',
+        max_signals: 100,
+        risk_score: 55,
+        risk_score_mapping: [],
+        language: 'kuery',
+        rule_id: 'query-rule-id',
+        interval: '5m',
+        exceptions_list: getListArrayMock(),
       },
       depth: 1,
     };
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
index 78818779dd661c..947938de6caca6 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
@@ -5,14 +5,14 @@
  */
 
 import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
-import { SignalSourceHit, Signal, Ancestor } from './types';
+import { Signal, Ancestor, BaseSignalHit } from './types';
 
 /**
  * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child
  * signal's `signal.parents` array.
  * @param doc The parent signal or event
  */
-export const buildParent = (doc: SignalSourceHit): Ancestor => {
+export const buildParent = (doc: BaseSignalHit): Ancestor => {
   if (doc._source.signal != null) {
     return {
       rule: doc._source.signal.rule.id,
@@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => {
  * creating an array of N+1 ancestors.
  * @param doc The parent signal/event for which to extend the ancestry.
  */
-export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => {
+export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => {
   const newAncestor = buildParent(doc);
   const existingAncestors = doc._source.signal?.ancestors;
   if (existingAncestors != null) {
@@ -53,7 +53,7 @@ export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => {
  * @param docs The parent signals/events of the new signal to be built.
  * @param rule The rule that is generating the new signal.
  */
-export const buildSignal = (docs: SignalSourceHit[], rule: Partial<RulesSchema>): Signal => {
+export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => {
   const parents = docs.map(buildParent);
   const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1;
   const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []);
@@ -70,7 +70,7 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial<RulesSchema>)
  * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event.
  * @param doc The parent signal/event of the new signal to be built.
  */
-export const additionalSignalFields = (doc: SignalSourceHit) => {
+export const additionalSignalFields = (doc: BaseSignalHit) => {
   return {
     parent: buildParent(doc),
     original_time: doc._source['@timestamp'],
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts
index 6ce0be54a9e7b6..522f4bfa5ef982 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts
@@ -90,7 +90,6 @@ export const getFilter = async ({
   };
 
   switch (type) {
-    case 'eql':
     case 'threat_match':
     case 'threshold': {
       return savedId != null ? savedQueryFilter() : queryFilter();
@@ -106,6 +105,9 @@ export const getFilter = async ({
         'Unsupported Rule of type "machine_learning" supplied to getFilter'
       );
     }
+    case 'eql': {
+      throw new BadRequestError('Unsupported Rule of type "eql" supplied to getFilter');
+    }
     default: {
       return assertUnreachable(type);
     }
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts
index c8f8341392553e..922fadb13a298c 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts
@@ -22,6 +22,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({
   author: [],
   buildingBlockType: null,
   description: 'Detecting root and admin users',
+  eventCategoryOverride: undefined,
   falsePositives: [],
   filters: null,
   from: 'now-6m',
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts
index dbb48d59d3a3f9..4006345b243856 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts
@@ -14,6 +14,7 @@ const signalSchema = schema.object({
   buildingBlockType: schema.nullable(schema.string()),
   description: schema.string(),
   note: schema.nullable(schema.string()),
+  eventCategoryOverride: schema.maybe(schema.string()),
   falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }),
   from: schema.string(),
   ruleId: schema.string(),
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 97ab12f9053585..f7b56f42755ab4 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -24,13 +24,19 @@ import { SetupPlugins } from '../../../plugin';
 import { getInputIndex } from './get_input_output_index';
 import { searchAfterAndBulkCreate } from './search_after_bulk_create';
 import { getFilter } from './get_filter';
-import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types';
+import {
+  SignalRuleAlertTypeDefinition,
+  RuleAlertAttributes,
+  EqlSignalSearchResponse,
+  BaseSignalHit,
+} from './types';
 import {
   getGapBetweenRuns,
   getListsClient,
   getExceptions,
   getGapMaxCatchupRatio,
   MAX_RULE_GAP_RATIO,
+  wrapSignal,
   createErrorsFromShard,
   createSearchAfterReturnType,
   mergeReturns,
@@ -50,6 +56,9 @@ import { ruleStatusServiceFactory } from './rule_status_service';
 import { buildRuleMessageFactory } from './rule_messages';
 import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client';
 import { getNotificationResultsLink } from '../notifications/utils';
+import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter';
+import { bulkInsertSignals } from './single_bulk_create';
+import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body';
 import { createThreatSignals } from './threat_mapping/create_threat_signals';
 
 export const signalRulesAlertType = ({
@@ -265,8 +274,6 @@ export const signalRulesAlertType = ({
               bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],
             }),
           ]);
-        } else if (isEqlRule(type)) {
-          throw new Error('EQL Rules are under development, execution is not yet implemented');
         } else if (isThresholdRule(type) && threshold) {
           const inputIndex = await getInputIndex(services, version, index);
           const esFilter = await getFilter({
@@ -378,7 +385,7 @@ export const signalRulesAlertType = ({
             buildRuleMessage,
             threatIndex,
           });
-        } else {
+        } else if (type === 'query' || type === 'saved_query') {
           const inputIndex = await getInputIndex(services, version, index);
           const esFilter = await getFilter({
             type,
@@ -417,6 +424,51 @@ export const signalRulesAlertType = ({
             throttle,
             buildRuleMessage,
           });
+        } else if (isEqlRule(type)) {
+          if (query === undefined) {
+            throw new Error('eql query rule must have a query defined');
+          }
+          const inputIndex = await getInputIndex(services, version, index);
+          const request = buildEqlSearchRequest(
+            query,
+            inputIndex,
+            params.from,
+            params.to,
+            searchAfterSize,
+            params.timestampOverride,
+            exceptionItems ?? [],
+            params.eventCategoryOverride
+          );
+          const response: EqlSignalSearchResponse = await services.callCluster(
+            'transport.request',
+            request
+          );
+          let newSignals: BaseSignalHit[] | undefined;
+          if (response.hits.sequences !== undefined) {
+            newSignals = response.hits.sequences.reduce(
+              (acc: BaseSignalHit[], sequence) =>
+                acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)),
+              []
+            );
+          } else if (response.hits.events !== undefined) {
+            newSignals = response.hits.events.map((event) =>
+              wrapSignal(buildSignalFromEvent(event, savedObject), outputIndex)
+            );
+          } else {
+            throw new Error(
+              'eql query response should have either `sequences` or `events` but had neither'
+            );
+          }
+          // TODO: replace with code that filters out recursive rule signals while allowing sequences and their building blocks
+          // const filteredSignals = filterDuplicateSignals(alertId, newSignals);
+          if (newSignals.length > 0) {
+            const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh);
+            result.bulkCreateTimes.push(insertResult.bulkCreateDuration);
+            result.createdSignalsCount += insertResult.createdItemsCount;
+          }
+          result.success = true;
+        } else {
+          throw new Error(`unknown rule type ${type}`);
         }
 
         if (result.success) {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
index e8f254e6a8966b..e3c3c940b3225b 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
@@ -7,7 +7,7 @@
 import { countBy, isEmpty } from 'lodash';
 import { performance } from 'perf_hooks';
 import { AlertServices } from '../../../../../alerts/server';
-import { SignalSearchResponse, BulkResponse } from './types';
+import { SignalSearchResponse, BulkResponse, SignalHit, BaseSignalHit } from './types';
 import { RuleAlertAction } from '../../../../common/detection_engine/types';
 import { RuleTypeParams, RefreshTypes } from '../types';
 import { generateId, makeFloatString, errorAggregator } from './utils';
@@ -59,6 +59,18 @@ export const filterDuplicateRules = (
   });
 };
 
+/**
+ * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched
+ * the detection query. This means we only have to compare the ruleId against the ancestors array.
+ * @param ruleId The rule id
+ * @param signals The candidate new signals
+ */
+export const filterDuplicateSignals = (ruleId: string, signals: SignalHit[]) => {
+  return signals.filter(
+    (doc) => !doc.signal.ancestors.some((ancestor) => ancestor.rule === ruleId)
+  );
+};
+
 export interface SingleBulkCreateResponse {
   success: boolean;
   bulkCreateDuration?: string;
@@ -66,6 +78,11 @@ export interface SingleBulkCreateResponse {
   errors: string[];
 }
 
+export interface BulkInsertSignalsResponse {
+  bulkCreateDuration: string;
+  createdItemsCount: number;
+}
+
 // Bulk Index documents.
 export const singleBulkCreate = async ({
   filteredEvents,
@@ -167,3 +184,46 @@ export const singleBulkCreate = async ({
     };
   }
 };
+
+// Bulk Index new signals.
+export const bulkInsertSignals = async (
+  signals: BaseSignalHit[],
+  logger: Logger,
+  services: AlertServices,
+  refresh: RefreshTypes
+): Promise<BulkInsertSignalsResponse> => {
+  // index documents after creating an ID based on the
+  // id and index of each parent and the rule ID
+  const bulkBody = signals.flatMap((doc) => [
+    {
+      create: {
+        _index: doc._index,
+        _id: doc._id,
+      },
+    },
+    doc._source,
+  ]);
+  const start = performance.now();
+  const response: BulkResponse = await services.callCluster('bulk', {
+    refresh,
+    body: bulkBody,
+  });
+  const end = performance.now();
+  logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`);
+  logger.debug(`took property says bulk took: ${response.took} milliseconds`);
+
+  if (response.errors) {
+    const duplicateSignalsCount = countBy(response.items, 'create.status')['409'];
+    logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`);
+    const errorCountByMessage = errorAggregator(response, [409]);
+    if (!isEmpty(errorCountByMessage)) {
+      logger.error(
+        `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}`
+      );
+    }
+  }
+
+  const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0;
+  logger.debug(`bulk created ${createdItemsCount} signals`);
+  return { bulkCreateDuration: makeFloatString(end - start), createdItemsCount };
+};
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
index 6ebdca0764e9d5..2f6ed0c1e3a8e9 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
@@ -16,7 +16,7 @@ import {
 } from '../../../../../alerts/server';
 import { RuleAlertAction } from '../../../../common/detection_engine/types';
 import { RuleTypeParams, RefreshTypes } from '../types';
-import { SearchResponse } from '../../types';
+import { SearchResponse, EqlSearchResponse, BaseHit } from '../../types';
 import { ListClient } from '../../../../../lists/server';
 import { Logger } from '../../../../../../../src/core/server';
 import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
@@ -53,6 +53,8 @@ export type SearchTypes =
 
 export interface SignalSource {
   [key: string]: SearchTypes;
+  // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not
+  // actually have @timestamp if a timestamp override is used
   '@timestamp': string;
   signal?: {
     // parent is deprecated: new signals should populate parents instead
@@ -60,6 +62,10 @@ export interface SignalSource {
     parent?: Ancestor;
     parents?: Ancestor[];
     ancestors: Ancestor[];
+    group?: {
+      id: string;
+      index?: number;
+    };
     rule: {
       id: string;
     };
@@ -116,6 +122,9 @@ export interface GetResponse {
 export type EventSearchResponse = SearchResponse<EventSource>;
 export type SignalSearchResponse = SearchResponse<SignalSource>;
 export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number];
+export type BaseSignalHit = BaseHit<SignalSource>;
+
+export type EqlSignalSearchResponse = EqlSearchResponse<SignalSource>;
 
 export type RuleExecutorOptions = Omit<AlertExecutorOptions, 'params'> & {
   params: RuleTypeParams;
@@ -140,11 +149,15 @@ export interface Ancestor {
 }
 
 export interface Signal {
-  rule: Partial<RulesSchema>;
+  rule: RulesSchema;
   // DEPRECATED: use parents instead of parent
   parent?: Ancestor;
   parents: Ancestor[];
   ancestors: Ancestor[];
+  group?: {
+    id: string;
+    index?: number;
+  };
   original_time?: string;
   original_event?: SearchTypes;
   status: Status;
@@ -155,7 +168,7 @@ export interface Signal {
 export interface SignalHit {
   '@timestamp': string;
   event: object;
-  signal: Partial<Signal>;
+  signal: Signal;
 }
 
 export interface AlertAttributes {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
index 97f3dbeaf44890..14e12b2ea46328 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
@@ -25,6 +25,8 @@ import {
   getListsClient,
   getSignalTimeTuples,
   getExceptions,
+  wrapBuildingBlocks,
+  generateSignalId,
   createErrorsFromShard,
   createSearchAfterReturnTypeFromResponse,
   createSearchAfterReturnType,
@@ -38,6 +40,7 @@ import {
   sampleBulkError,
   sampleBulkErrorItem,
   mockLogger,
+  sampleSignalHit,
   sampleDocSearchResultsWithSortId,
   sampleEmptyDocSearchResults,
   sampleDocSearchResultsNoSortIdNoHits,
@@ -794,6 +797,56 @@ describe('utils', () => {
     });
   });
 
+  describe('wrapBuildingBlocks', () => {
+    it('should generate a unique id for each building block', () => {
+      const wrappedBlocks = wrapBuildingBlocks(
+        [sampleSignalHit(), sampleSignalHit()],
+        'test-index'
+      );
+      const blockIds: string[] = [];
+      wrappedBlocks.forEach((block) => {
+        expect(blockIds.includes(block._id)).toEqual(false);
+        blockIds.push(block._id);
+      });
+    });
+
+    it('should generate different ids for identical documents in different sequences', () => {
+      const wrappedBlockSequence1 = wrapBuildingBlocks([sampleSignalHit()], 'test-index');
+      const wrappedBlockSequence2 = wrapBuildingBlocks(
+        [sampleSignalHit(), sampleSignalHit()],
+        'test-index'
+      );
+      const blockId = wrappedBlockSequence1[0]._id;
+      wrappedBlockSequence2.forEach((block) => {
+        expect(block._id).not.toEqual(blockId);
+      });
+    });
+
+    it('should generate the same ids when given the same sequence twice', () => {
+      const wrappedBlockSequence1 = wrapBuildingBlocks(
+        [sampleSignalHit(), sampleSignalHit()],
+        'test-index'
+      );
+      const wrappedBlockSequence2 = wrapBuildingBlocks(
+        [sampleSignalHit(), sampleSignalHit()],
+        'test-index'
+      );
+      wrappedBlockSequence1.forEach((block, idx) => {
+        expect(block._id).toEqual(wrappedBlockSequence2[idx]._id);
+      });
+    });
+  });
+
+  describe('generateSignalId', () => {
+    it('generates a unique signal id for same signal with different rule id', () => {
+      const signalId1 = generateSignalId(sampleSignalHit().signal);
+      const modifiedSignal = sampleSignalHit();
+      modifiedSignal.signal.rule.id = 'some other rule id';
+      const signalIdModified = generateSignalId(modifiedSignal.signal);
+      expect(signalId1).not.toEqual(signalIdModified);
+    });
+  });
+
   describe('createErrorsFromShard', () => {
     test('empty errors will return an empty array', () => {
       const createdErrors = createErrorsFromShard({ errors: [] });
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
index 2eabc03dccad71..53089b7f1ca2b8 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
@@ -16,8 +16,11 @@ import {
   BulkResponse,
   BulkResponseErrorAggregation,
   isValidUnit,
+  SignalHit,
+  BaseSignalHit,
   SearchAfterAndBulkCreateReturnType,
   SignalSearchResponse,
+  Signal,
 } from './types';
 import { BuildRuleMessage } from './rule_messages';
 import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates';
@@ -212,6 +215,60 @@ export const generateId = (
   ruleId: string
 ): string => createHash('sha256').update(docIndex.concat(docId, version, ruleId)).digest('hex');
 
+// TODO: do we need to include version in the id? If it does matter then we should include it in signal.parents as well
+export const generateSignalId = (signal: Signal) =>
+  createHash('sha256')
+    .update(
+      signal.parents
+        .reduce((acc, parent) => acc.concat(parent.id, parent.index), '')
+        .concat(signal.rule.id)
+    )
+    .digest('hex');
+
+/**
+ * Generates unique doc ids for each building block signal within a sequence. The id of each building block
+ * depends on the parents of every building block, so that a signal which appears in multiple different sequences
+ * (e.g. if multiple rules build sequences that share a common event/signal) will get a unique id per sequence.
+ * @param buildingBlocks The full list of building blocks in the sequence.
+ */
+export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] => {
+  const baseHashString = buildingBlocks.reduce(
+    (baseString, block) =>
+      baseString
+        .concat(
+          block.signal.parents.reduce((acc, parent) => acc.concat(parent.id, parent.index), '')
+        )
+        .concat(block.signal.rule.id),
+    ''
+  );
+  return buildingBlocks.map((block, idx) =>
+    createHash('sha256').update(baseHashString).update(String(idx)).digest('hex')
+  );
+};
+
+export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => {
+  const blockIds = generateBuildingBlockIds(buildingBlocks);
+  return buildingBlocks.map((block, idx) => {
+    return {
+      _id: blockIds[idx],
+      _index: index,
+      _source: {
+        ...block,
+      },
+    };
+  });
+};
+
+export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => {
+  return {
+    _id: generateSignalId(signal.signal),
+    _index: index,
+    _source: {
+      ...signal,
+    },
+  };
+};
+
 export const parseInterval = (intervalString: string): moment.Duration | null => {
   try {
     return moment.duration(parseDuration(intervalString));
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts
index b0554adcc46b0f..728f5b1dd867fd 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts
@@ -37,6 +37,7 @@ import {
   SeverityMappingOrUndefined,
   TimestampOverrideOrUndefined,
   Type,
+  EventCategoryOverrideOrUndefined,
 } from '../../../common/detection_engine/schemas/common/schemas';
 import {
   ThreatIndexOrUndefined,
@@ -56,6 +57,7 @@ export interface RuleTypeParams {
   buildingBlockType: BuildingBlockTypeOrUndefined;
   description: Description;
   note: NoteOrUndefined;
+  eventCategoryOverride: EventCategoryOverrideOrUndefined;
   falsePositives: FalsePositives;
   from: From;
   ruleId: RuleId;
diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts
index 7e59280cd1358d..117cffd844cfb1 100644
--- a/x-pack/plugins/security_solution/server/lib/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/types.ts
@@ -16,6 +16,7 @@ import { Sources } from './sources';
 import { Note } from './note/saved_object';
 import { PinnedEvent } from './pinned_event/saved_object';
 import { Timeline } from './timeline/saved_object';
+import { SearchTypes } from './detection_engine/signals/types';
 
 export * from './hosts';
 
@@ -44,6 +45,12 @@ export interface TotalValue {
   relation: string;
 }
 
+export interface BaseHit<T> {
+  _index: string;
+  _id: string;
+  _source: T;
+}
+
 export interface SearchResponse<T> {
   took: number;
   timed_out: boolean;
@@ -52,27 +59,43 @@ export interface SearchResponse<T> {
   hits: {
     total: TotalValue | number;
     max_score: number;
-    hits: Array<{
-      _index: string;
-      _type: string;
-      _id: string;
-      _score: number;
-      _source: T;
-      _version?: number;
-      _explanation?: Explanation;
-      fields?: string[];
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      highlight?: any;
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      inner_hits?: any;
-      matched_queries?: string[];
-      sort?: string[];
-    }>;
+    hits: Array<
+      BaseHit<T> & {
+        _type: string;
+        _score: number;
+        _version?: number;
+        _explanation?: Explanation;
+        fields?: string[];
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        highlight?: any;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        inner_hits?: any;
+        matched_queries?: string[];
+        sort?: string[];
+      }
+    >;
   };
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   aggregations?: any;
 }
 
+export interface EqlSequence<T> {
+  join_keys: SearchTypes[];
+  events: Array<BaseHit<T>>;
+}
+
+export interface EqlSearchResponse<T> {
+  is_partial: boolean;
+  is_running: boolean;
+  took: number;
+  timed_out: boolean;
+  hits: {
+    total: TotalValue;
+    sequences?: Array<EqlSequence<T>>;
+    events?: Array<BaseHit<T>>;
+  };
+}
+
 export interface ShardsResponse {
   total: number;
   successful: number;

From 3f6c0d688cf38d5b20db788b7a51231ebd95e7c6 Mon Sep 17 00:00:00 2001
From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com>
Date: Mon, 28 Sep 2020 12:43:34 +0300
Subject: [PATCH 16/22] Lazy load metric & mardown visualizations (#78391)

* Lazy load metrics vis

* Use common chart spinner

* Simplify markdown renderer

* Update tests

* Update types for metric vis

* Fix tests

* Fix merge conflict

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 src/plugins/vis_type_markdown/kibana.json     |  2 +-
 .../__snapshots__/markdown_fn.test.ts.snap    |  2 +-
 .../vis_type_markdown/public/index.scss       |  8 --
 .../vis_type_markdown/public/markdown_fn.ts   |  8 +-
 .../public/markdown_renderer.tsx              | 50 ++++------
 .../{_markdown_vis.scss => markdown_vis.scss} |  7 ++
 .../public/markdown_vis_controller.test.tsx   | 46 ++--------
 .../public/markdown_vis_controller.tsx        | 92 +++++--------------
 .../vis_type_markdown/public/plugin.ts        |  8 +-
 .../metric_vis.scss}                          |  7 ++
 .../components/metric_vis_component.test.tsx  |  2 +-
 .../components/metric_vis_component.tsx       | 10 +-
 src/plugins/vis_type_metric/public/index.scss |  8 --
 src/plugins/vis_type_metric/public/index.ts   |  1 -
 .../vis_type_metric/public/metric_vis_fn.ts   |  4 +-
 .../public/metric_vis_renderer.tsx            | 34 +++----
 src/plugins/vis_type_metric/public/plugin.ts  |  3 +-
 .../vis_type_metric/public/services.ts        |  3 -
 .../public/components/_visualization.scss     |  7 ++
 .../components/visualization_container.tsx    | 29 +++++-
 20 files changed, 132 insertions(+), 199 deletions(-)
 delete mode 100644 src/plugins/vis_type_markdown/public/index.scss
 rename src/plugins/vis_type_markdown/public/{_markdown_vis.scss => markdown_vis.scss} (55%)
 rename src/plugins/vis_type_metric/public/{_metric_vis.scss => components/metric_vis.scss} (78%)
 delete mode 100644 src/plugins/vis_type_metric/public/index.scss

diff --git a/src/plugins/vis_type_markdown/kibana.json b/src/plugins/vis_type_markdown/kibana.json
index 5723fdefe1e4cf..c0afcb0e99d139 100644
--- a/src/plugins/vis_type_markdown/kibana.json
+++ b/src/plugins/vis_type_markdown/kibana.json
@@ -4,5 +4,5 @@
     "ui": true,
     "server": true,
     "requiredPlugins": ["expressions", "visualizations"],
-    "requiredBundles": ["kibanaUtils", "kibanaReact", "charts", "visualizations", "expressions", "visDefaultEditor"]
+    "requiredBundles": ["kibanaReact", "charts", "visualizations", "expressions", "visDefaultEditor"]
 }
diff --git a/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap b/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap
index 473e2cba742b7a..9983f67d4be4da 100644
--- a/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap
+++ b/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap
@@ -5,7 +5,7 @@ Object {
   "as": "markdown_vis",
   "type": "render",
   "value": Object {
-    "visConfig": Object {
+    "visParams": Object {
       "fontSize": 12,
       "markdown": "## hello _markdown_",
       "openLinksInNewTab": true,
diff --git a/src/plugins/vis_type_markdown/public/index.scss b/src/plugins/vis_type_markdown/public/index.scss
deleted file mode 100644
index ddb7fe3a6b0d9f..00000000000000
--- a/src/plugins/vis_type_markdown/public/index.scss
+++ /dev/null
@@ -1,8 +0,0 @@
-// Prefix all styles with "mkd" to avoid conflicts.
-// Examples
-//   mkdChart
-//   mkdChart__legend
-//   mkdChart__legend--small
-//   mkdChart__legend-isLoading
-
-@import './markdown_vis';
diff --git a/src/plugins/vis_type_markdown/public/markdown_fn.ts b/src/plugins/vis_type_markdown/public/markdown_fn.ts
index 4b3c9989431f9b..eaa2c840f80466 100644
--- a/src/plugins/vis_type_markdown/public/markdown_fn.ts
+++ b/src/plugins/vis_type_markdown/public/markdown_fn.ts
@@ -21,16 +21,16 @@ import { i18n } from '@kbn/i18n';
 import { ExpressionFunctionDefinition, Render } from '../../expressions/public';
 import { Arguments, MarkdownVisParams } from './types';
 
-interface RenderValue {
+export interface MarkdownVisRenderValue {
   visType: 'markdown';
-  visConfig: MarkdownVisParams;
+  visParams: MarkdownVisParams;
 }
 
 export type MarkdownVisExpressionFunctionDefinition = ExpressionFunctionDefinition<
   'markdownVis',
   unknown,
   Arguments,
-  Render<RenderValue>
+  Render<MarkdownVisRenderValue>
 >;
 
 export const createMarkdownVisFn = (): MarkdownVisExpressionFunctionDefinition => ({
@@ -70,7 +70,7 @@ export const createMarkdownVisFn = (): MarkdownVisExpressionFunctionDefinition =
       as: 'markdown_vis',
       value: {
         visType: 'markdown',
-        visConfig: {
+        visParams: {
           markdown: args.markdown,
           openLinksInNewTab: args.openLinksInNewTab,
           fontSize: parseInt(args.font.spec.fontSize || '12', 10),
diff --git a/src/plugins/vis_type_markdown/public/markdown_renderer.tsx b/src/plugins/vis_type_markdown/public/markdown_renderer.tsx
index 5950a762635b28..8071196c6a213d 100644
--- a/src/plugins/vis_type_markdown/public/markdown_renderer.tsx
+++ b/src/plugins/vis_type_markdown/public/markdown_renderer.tsx
@@ -17,41 +17,29 @@
  * under the License.
  */
 
-import React from 'react';
+import React, { lazy } from 'react';
 import { render, unmountComponentAtNode } from 'react-dom';
 import { VisualizationContainer } from '../../visualizations/public';
 import { ExpressionRenderDefinition } from '../../expressions/common/expression_renderers';
-import { MarkdownVisWrapper } from './markdown_vis_controller';
-import { StartServicesGetter } from '../../kibana_utils/public';
+import { MarkdownVisRenderValue } from './markdown_fn';
 
-export const getMarkdownRenderer = (start: StartServicesGetter) => {
-  const markdownVisRenderer: () => ExpressionRenderDefinition = () => ({
-    name: 'markdown_vis',
-    displayName: 'markdown visualization',
-    reuseDomNode: true,
-    render: async (domNode: HTMLElement, config: any, handlers: any) => {
-      const { visConfig } = config;
+// @ts-ignore
+const MarkdownVisComponent = lazy(() => import('./markdown_vis_controller'));
 
-      const I18nContext = await start().core.i18n.Context;
+export const markdownVisRenderer: ExpressionRenderDefinition<MarkdownVisRenderValue> = {
+  name: 'markdown_vis',
+  displayName: 'markdown visualization',
+  reuseDomNode: true,
+  render: async (domNode, { visParams }, handlers) => {
+    handlers.onDestroy(() => {
+      unmountComponentAtNode(domNode);
+    });
 
-      handlers.onDestroy(() => {
-        unmountComponentAtNode(domNode);
-      });
-
-      render(
-        <I18nContext>
-          <VisualizationContainer className="markdownVis">
-            <MarkdownVisWrapper
-              visParams={visConfig}
-              renderComplete={handlers.done}
-              fireEvent={handlers.event}
-            />
-          </VisualizationContainer>
-        </I18nContext>,
-        domNode
-      );
-    },
-  });
-
-  return markdownVisRenderer;
+    render(
+      <VisualizationContainer className="markdownVis">
+        <MarkdownVisComponent {...visParams} renderComplete={handlers.done} />
+      </VisualizationContainer>,
+      domNode
+    );
+  },
 };
diff --git a/src/plugins/vis_type_markdown/public/_markdown_vis.scss b/src/plugins/vis_type_markdown/public/markdown_vis.scss
similarity index 55%
rename from src/plugins/vis_type_markdown/public/_markdown_vis.scss
rename to src/plugins/vis_type_markdown/public/markdown_vis.scss
index fb0a3d05e5e857..2356562a86ed0a 100644
--- a/src/plugins/vis_type_markdown/public/_markdown_vis.scss
+++ b/src/plugins/vis_type_markdown/public/markdown_vis.scss
@@ -1,3 +1,10 @@
+// Prefix all styles with "mkd" to avoid conflicts.
+// Examples
+//   mkdChart
+//   mkdChart__legend
+//   mkdChart__legend--small
+//   mkdChart__legend-isLoading
+
 .mkdVis {
   padding: $euiSizeS;
   width: 100%;
diff --git a/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx b/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx
index 6df205b21d910a..36850fc820ded5 100644
--- a/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx
+++ b/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx
@@ -20,7 +20,7 @@
 import React from 'react';
 import { wait } from '@testing-library/dom';
 import { render, cleanup } from '@testing-library/react/pure';
-import { MarkdownVisWrapper } from './markdown_vis_controller';
+import MarkdownVisComponent from './markdown_vis_controller';
 
 afterEach(cleanup);
 
@@ -36,7 +36,7 @@ describe('markdown vis controller', () => {
     };
 
     const { getByTestId, getByText } = render(
-      <MarkdownVisWrapper visParams={vis.params} renderComplete={jest.fn()} fireEvent={jest.fn()} />
+      <MarkdownVisComponent {...vis.params} renderComplete={jest.fn()} />
     );
 
     await wait(() => getByTestId('markdownBody'));
@@ -60,7 +60,7 @@ describe('markdown vis controller', () => {
     };
 
     const { getByTestId, getByText } = render(
-      <MarkdownVisWrapper visParams={vis.params} renderComplete={jest.fn()} fireEvent={jest.fn()} />
+      <MarkdownVisComponent {...vis.params} renderComplete={jest.fn()} />
     );
 
     await wait(() => getByTestId('markdownBody'));
@@ -82,7 +82,7 @@ describe('markdown vis controller', () => {
     };
 
     const { getByTestId, getByText, rerender } = render(
-      <MarkdownVisWrapper visParams={vis.params} renderComplete={jest.fn()} fireEvent={jest.fn()} />
+      <MarkdownVisComponent {...vis.params} renderComplete={jest.fn()} />
     );
 
     await wait(() => getByTestId('markdownBody'));
@@ -90,9 +90,7 @@ describe('markdown vis controller', () => {
     expect(getByText(/initial/i)).toBeInTheDocument();
 
     vis.params.markdown = 'Updated';
-    rerender(
-      <MarkdownVisWrapper visParams={vis.params} renderComplete={jest.fn()} fireEvent={jest.fn()} />
-    );
+    rerender(<MarkdownVisComponent {...vis.params} renderComplete={jest.fn()} />);
 
     expect(getByText(/Updated/i)).toBeInTheDocument();
   });
@@ -114,11 +112,7 @@ describe('markdown vis controller', () => {
 
     it('should be called on initial rendering', async () => {
       const { getByTestId } = render(
-        <MarkdownVisWrapper
-          visParams={vis.params}
-          renderComplete={renderComplete}
-          fireEvent={jest.fn()}
-        />
+        <MarkdownVisComponent {...vis.params} renderComplete={renderComplete} />
       );
 
       await wait(() => getByTestId('markdownBody'));
@@ -128,11 +122,7 @@ describe('markdown vis controller', () => {
 
     it('should be called on successive render when params change', async () => {
       const { getByTestId, rerender } = render(
-        <MarkdownVisWrapper
-          visParams={vis.params}
-          renderComplete={renderComplete}
-          fireEvent={jest.fn()}
-        />
+        <MarkdownVisComponent {...vis.params} renderComplete={renderComplete} />
       );
 
       await wait(() => getByTestId('markdownBody'));
@@ -142,24 +132,14 @@ describe('markdown vis controller', () => {
       renderComplete.mockClear();
       vis.params.markdown = 'changed';
 
-      rerender(
-        <MarkdownVisWrapper
-          visParams={vis.params}
-          renderComplete={renderComplete}
-          fireEvent={jest.fn()}
-        />
-      );
+      rerender(<MarkdownVisComponent {...vis.params} renderComplete={renderComplete} />);
 
       expect(renderComplete).toHaveBeenCalledTimes(1);
     });
 
     it('should be called on successive render even without data change', async () => {
       const { getByTestId, rerender } = render(
-        <MarkdownVisWrapper
-          visParams={vis.params}
-          renderComplete={renderComplete}
-          fireEvent={jest.fn()}
-        />
+        <MarkdownVisComponent {...vis.params} renderComplete={renderComplete} />
       );
 
       await wait(() => getByTestId('markdownBody'));
@@ -168,13 +148,7 @@ describe('markdown vis controller', () => {
 
       renderComplete.mockClear();
 
-      rerender(
-        <MarkdownVisWrapper
-          visParams={vis.params}
-          renderComplete={renderComplete}
-          fireEvent={jest.fn()}
-        />
-      );
+      rerender(<MarkdownVisComponent {...vis.params} renderComplete={renderComplete} />);
 
       expect(renderComplete).toHaveBeenCalledTimes(1);
     });
diff --git a/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx b/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx
index e1155ca42df723..a2387b96eab6d6 100644
--- a/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx
+++ b/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx
@@ -17,83 +17,35 @@
  * under the License.
  */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import { Markdown } from '../../kibana_react/public';
 import { MarkdownVisParams } from './types';
 
+import './markdown_vis.scss';
+
 interface MarkdownVisComponentProps extends MarkdownVisParams {
   renderComplete: () => void;
 }
 
-/**
- * The MarkdownVisComponent renders markdown to HTML and presents it.
- */
-class MarkdownVisComponent extends React.Component<MarkdownVisComponentProps> {
-  /**
-   * Will be called after the first render when the component is present in the DOM.
-   *
-   * We call renderComplete here, to signal, that we are done with rendering.
-   */
-  componentDidMount() {
-    this.props.renderComplete();
-  }
-
-  /**
-   * Will be called after the component has been updated and the changes has been
-   * flushed into the DOM.
-   *
-   * We will use this to signal that we are done rendering by calling the
-   * renderComplete property.
-   */
-  componentDidUpdate() {
-    this.props.renderComplete();
-  }
+const MarkdownVisComponent = ({
+  fontSize,
+  markdown,
+  openLinksInNewTab,
+  renderComplete,
+}: MarkdownVisComponentProps) => {
+  useEffect(renderComplete); // renderComplete will be called after each render to signal, that we are done with rendering.
 
-  /**
-   * Render the actual HTML.
-   * Note: if only fontSize parameter has changed, this method will be called
-   * and return the appropriate JSX, but React will detect, that only the
-   * style argument has been updated, and thus only set this attribute to the DOM.
-   */
-  render() {
-    return (
-      <div className="mkdVis" style={{ fontSize: `${this.props.fontSize}pt` }}>
-        <Markdown
-          data-test-subj="markdownBody"
-          markdown={this.props.markdown}
-          openLinksInNewTab={this.props.openLinksInNewTab}
-        />
-      </div>
-    );
-  }
-}
-
-/**
- * This is a wrapper component, that is actually used as the visualization.
- * The sole purpose of this component is to extract all required parameters from
- * the properties and pass them down as separate properties to the actual component.
- * That way the actual (MarkdownVisComponent) will properly trigger it's prop update
- * callback (componentWillReceiveProps) if one of these params change. It wouldn't
- * trigger otherwise (e.g. it doesn't for this wrapper), since it only triggers
- * if the reference to the prop changes (in this case the reference to vis).
- *
- * The way React works, this wrapper nearly brings no overhead, but allows us
- * to use proper lifecycle methods in the actual component.
- */
-
-export interface MarkdownVisWrapperProps {
-  visParams: MarkdownVisParams;
-  fireEvent: (event: any) => void;
-  renderComplete: () => void;
-}
-
-export function MarkdownVisWrapper(props: MarkdownVisWrapperProps) {
   return (
-    <MarkdownVisComponent
-      fontSize={props.visParams.fontSize}
-      markdown={props.visParams.markdown}
-      openLinksInNewTab={props.visParams.openLinksInNewTab}
-      renderComplete={props.renderComplete}
-    />
+    <div className="mkdVis" style={{ fontSize: `${fontSize}pt` }}>
+      <Markdown
+        data-test-subj="markdownBody"
+        markdown={markdown}
+        openLinksInNewTab={openLinksInNewTab}
+      />
+    </div>
   );
-}
+};
+
+// default export required for React.Lazy
+// eslint-disable-next-line import/no-default-export
+export { MarkdownVisComponent as default };
diff --git a/src/plugins/vis_type_markdown/public/plugin.ts b/src/plugins/vis_type_markdown/public/plugin.ts
index c117df7e0fa33c..790b19876d366d 100644
--- a/src/plugins/vis_type_markdown/public/plugin.ts
+++ b/src/plugins/vis_type_markdown/public/plugin.ts
@@ -24,10 +24,7 @@ import { VisualizationsSetup } from '../../visualizations/public';
 import { markdownVisDefinition } from './markdown_vis';
 import { createMarkdownVisFn } from './markdown_fn';
 import { ConfigSchema } from '../config';
-
-import './index.scss';
-import { getMarkdownRenderer } from './markdown_renderer';
-import { createStartServicesGetter } from '../../kibana_utils/public';
+import { markdownVisRenderer } from './markdown_renderer';
 
 /** @internal */
 export interface MarkdownPluginSetupDependencies {
@@ -44,9 +41,8 @@ export class MarkdownPlugin implements Plugin<void, void> {
   }
 
   public setup(core: CoreSetup, { expressions, visualizations }: MarkdownPluginSetupDependencies) {
-    const start = createStartServicesGetter(core.getStartServices);
     visualizations.createBaseVisualization(markdownVisDefinition);
-    expressions.registerRenderer(getMarkdownRenderer(start));
+    expressions.registerRenderer(markdownVisRenderer);
     expressions.registerFunction(createMarkdownVisFn);
   }
 
diff --git a/src/plugins/vis_type_metric/public/_metric_vis.scss b/src/plugins/vis_type_metric/public/components/metric_vis.scss
similarity index 78%
rename from src/plugins/vis_type_metric/public/_metric_vis.scss
rename to src/plugins/vis_type_metric/public/components/metric_vis.scss
index b1f04cc93c4b7d..5665ba8e8d0994 100644
--- a/src/plugins/vis_type_metric/public/_metric_vis.scss
+++ b/src/plugins/vis_type_metric/public/components/metric_vis.scss
@@ -1,3 +1,10 @@
+// Prefix all styles with "mtr" to avoid conflicts.
+// Examples
+//   mtrChart
+//   mtrChart__legend
+//   mtrChart__legend--small
+//   mtrChart__legend-isLoading
+
 .mtrVis {
   width: 100%;
   display: flex;
diff --git a/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx b/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx
index b56d4e4f62e416..7f82c6adb56943 100644
--- a/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx
+++ b/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx
@@ -20,7 +20,7 @@
 import React from 'react';
 import { shallow } from 'enzyme';
 
-import { MetricVisComponent, MetricVisComponentProps } from './metric_vis_component';
+import MetricVisComponent, { MetricVisComponentProps } from './metric_vis_component';
 
 jest.mock('../services', () => ({
   getFormatService: () => ({
diff --git a/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx b/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx
index 9ce3820ee4e23e..e5c7db65c09a88 100644
--- a/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx
+++ b/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx
@@ -30,14 +30,16 @@ import { getFormatService } from '../services';
 import { SchemaConfig } from '../../../visualizations/public';
 import { Range } from '../../../expressions/public';
 
+import './metric_vis.scss';
+
 export interface MetricVisComponentProps {
-  visParams: VisParams;
+  visParams: Pick<VisParams, 'metric' | 'dimensions'>;
   visData: Input;
   fireEvent: (event: any) => void;
   renderComplete: () => void;
 }
 
-export class MetricVisComponent extends Component<MetricVisComponentProps> {
+class MetricVisComponent extends Component<MetricVisComponentProps> {
   private getLabels() {
     const config = this.props.visParams.metric;
     const isPercentageMode = config.percentageMode;
@@ -209,3 +211,7 @@ export class MetricVisComponent extends Component<MetricVisComponentProps> {
     return metricsHtml;
   }
 }
+
+// default export required for React.Lazy
+// eslint-disable-next-line import/no-default-export
+export { MetricVisComponent as default };
diff --git a/src/plugins/vis_type_metric/public/index.scss b/src/plugins/vis_type_metric/public/index.scss
deleted file mode 100644
index 638f9ac1ef93ad..00000000000000
--- a/src/plugins/vis_type_metric/public/index.scss
+++ /dev/null
@@ -1,8 +0,0 @@
-// Prefix all styles with "mtr" to avoid conflicts.
-// Examples
-//   mtrChart
-//   mtrChart__legend
-//   mtrChart__legend--small
-//   mtrChart__legend-isLoading
-
-@import 'metric_vis';
diff --git a/src/plugins/vis_type_metric/public/index.ts b/src/plugins/vis_type_metric/public/index.ts
index 3d3e1879a51d93..ac541a9577cfc4 100644
--- a/src/plugins/vis_type_metric/public/index.ts
+++ b/src/plugins/vis_type_metric/public/index.ts
@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import './index.scss';
 import { PluginInitializerContext } from 'kibana/public';
 import { MetricVisPlugin as Plugin } from './plugin';
 
diff --git a/src/plugins/vis_type_metric/public/metric_vis_fn.ts b/src/plugins/vis_type_metric/public/metric_vis_fn.ts
index b58be635817241..97b1e6822333ee 100644
--- a/src/plugins/vis_type_metric/public/metric_vis_fn.ts
+++ b/src/plugins/vis_type_metric/public/metric_vis_fn.ts
@@ -46,7 +46,7 @@ interface Arguments {
   bucket: any; // these aren't typed yet
 }
 
-interface RenderValue {
+export interface MetricVisRenderValue {
   visType: typeof visType;
   visData: Input;
   visConfig: Pick<VisParams, 'metric' | 'dimensions'>;
@@ -57,7 +57,7 @@ export type MetricVisExpressionFunctionDefinition = ExpressionFunctionDefinition
   'metricVis',
   Input,
   Arguments,
-  Render<RenderValue>
+  Render<MetricVisRenderValue>
 >;
 
 export const createMetricVisFn = (): MetricVisExpressionFunctionDefinition => ({
diff --git a/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx b/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx
index 2bae668b080eaf..bf0d6da9fba059 100644
--- a/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx
+++ b/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx
@@ -17,37 +17,33 @@
  * under the License.
  */
 
-import React from 'react';
+import React, { lazy } from 'react';
 import { render, unmountComponentAtNode } from 'react-dom';
-import { MetricVisComponent } from './components/metric_vis_component';
-import { getI18n } from './services';
+
 import { VisualizationContainer } from '../../visualizations/public';
 import { ExpressionRenderDefinition } from '../../expressions/common/expression_renderers';
+import { MetricVisRenderValue } from './metric_vis_fn';
+// @ts-ignore
+const MetricVisComponent = lazy(() => import('./components/metric_vis_component'));
 
-export const metricVisRenderer: () => ExpressionRenderDefinition = () => ({
+export const metricVisRenderer: () => ExpressionRenderDefinition<MetricVisRenderValue> = () => ({
   name: 'metric_vis',
   displayName: 'metric visualization',
   reuseDomNode: true,
-  render: async (domNode: HTMLElement, config: any, handlers: any) => {
-    const { visData, visConfig } = config;
-
-    const I18nContext = getI18n().Context;
-
+  render: async (domNode, { visData, visConfig }, handlers) => {
     handlers.onDestroy(() => {
       unmountComponentAtNode(domNode);
     });
 
     render(
-      <I18nContext>
-        <VisualizationContainer className="mtrVis">
-          <MetricVisComponent
-            visData={visData}
-            visParams={visConfig}
-            renderComplete={handlers.done}
-            fireEvent={handlers.event}
-          />
-        </VisualizationContainer>
-      </I18nContext>,
+      <VisualizationContainer className="mtrVis" showNoResult={!visData.rows?.length}>
+        <MetricVisComponent
+          visData={visData}
+          visParams={visConfig}
+          renderComplete={handlers.done}
+          fireEvent={handlers.event}
+        />
+      </VisualizationContainer>,
       domNode
     );
   },
diff --git a/src/plugins/vis_type_metric/public/plugin.ts b/src/plugins/vis_type_metric/public/plugin.ts
index b9e094aa76889f..c653d1bdaf965c 100644
--- a/src/plugins/vis_type_metric/public/plugin.ts
+++ b/src/plugins/vis_type_metric/public/plugin.ts
@@ -25,7 +25,7 @@ import { createMetricVisFn } from './metric_vis_fn';
 import { createMetricVisTypeDefinition } from './metric_vis_type';
 import { ChartsPluginSetup } from '../../charts/public';
 import { DataPublicPluginStart } from '../../data/public';
-import { setFormatService, setI18n } from './services';
+import { setFormatService } from './services';
 import { ConfigSchema } from '../config';
 import { metricVisRenderer } from './metric_vis_renderer';
 
@@ -59,7 +59,6 @@ export class MetricVisPlugin implements Plugin<void, void> {
   }
 
   public start(core: CoreStart, { data }: MetricVisPluginStartDependencies) {
-    setI18n(core.i18n);
     setFormatService(data.fieldFormats);
   }
 }
diff --git a/src/plugins/vis_type_metric/public/services.ts b/src/plugins/vis_type_metric/public/services.ts
index 0e19cfdce228d4..681afbaf0b268a 100644
--- a/src/plugins/vis_type_metric/public/services.ts
+++ b/src/plugins/vis_type_metric/public/services.ts
@@ -17,12 +17,9 @@
  * under the License.
  */
 
-import { I18nStart } from 'kibana/public';
 import { createGetterSetter } from '../../kibana_utils/common';
 import { DataPublicPluginStart } from '../../data/public';
 
 export const [getFormatService, setFormatService] = createGetterSetter<
   DataPublicPluginStart['fieldFormats']
 >('metric data.fieldFormats');
-
-export const [getI18n, setI18n] = createGetterSetter<I18nStart>('I18n');
diff --git a/src/plugins/visualizations/public/components/_visualization.scss b/src/plugins/visualizations/public/components/_visualization.scss
index 785968d2883e76..f5e2d4fcf2862d 100644
--- a/src/plugins/visualizations/public/components/_visualization.scss
+++ b/src/plugins/visualizations/public/components/_visualization.scss
@@ -70,3 +70,10 @@
   flex-direction: column;
 }
 
+.visChart__spinner {
+  display: flex;
+  flex: 1 1 auto;
+  justify-content: center;
+  align-items: center;
+}
+
diff --git a/src/plugins/visualizations/public/components/visualization_container.tsx b/src/plugins/visualizations/public/components/visualization_container.tsx
index d6f87d4cea123e..007a9e6e9dde4f 100644
--- a/src/plugins/visualizations/public/components/visualization_container.tsx
+++ b/src/plugins/visualizations/public/components/visualization_container.tsx
@@ -17,14 +17,35 @@
  * under the License.
  */
 
-import React, { ReactNode } from 'react';
+import React, { ReactNode, Suspense } from 'react';
+import { EuiLoadingChart } from '@elastic/eui';
+import classNames from 'classnames';
+import { VisualizationNoResults } from './visualization_noresults';
 
 interface VisualizationContainerProps {
   className?: string;
   children: ReactNode;
+  showNoResult?: boolean;
 }
 
-export const VisualizationContainer = (props: VisualizationContainerProps) => {
-  const classes = `visualization ${props.className}`;
-  return <div className={classes}>{props.children}</div>;
+export const VisualizationContainer = ({
+  className,
+  children,
+  showNoResult = false,
+}: VisualizationContainerProps) => {
+  const classes = classNames('visualization', className);
+
+  const fallBack = (
+    <div className="visChart__spinner">
+      <EuiLoadingChart mono size="l" />
+    </div>
+  );
+
+  return (
+    <div className={classes}>
+      <Suspense fallback={fallBack}>
+        {showNoResult ? <VisualizationNoResults /> : children}
+      </Suspense>
+    </div>
+  );
 };

From a71d0693dc45d1e9eb186eb59557107c85f1f393 Mon Sep 17 00:00:00 2001
From: Tim Roes <tim.roes@elastic.co>
Date: Mon, 28 Sep 2020 14:33:48 +0200
Subject: [PATCH 17/22] TypeScript cleanup in visualizations plugin (#78428)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../public/input_control_vis_type.ts             |  5 ++++-
 .../input_control_vis/public/vis_controller.tsx  | 12 ++++++------
 .../vis_type_metric/public/metric_vis_type.ts    |  3 ++-
 .../public/table_vis_controller.test.ts          |  2 +-
 .../vis_type_table/public/table_vis_type.ts      |  8 +++++---
 .../vis_type_table/public/vis_controller.ts      |  2 +-
 .../vis_type_vislib/public/vis_controller.tsx    |  2 +-
 src/plugins/visualizations/public/index.ts       |  2 +-
 src/plugins/visualizations/public/types.ts       |  9 +++++++--
 .../public/vis_types/base_vis_type.ts            | 16 +++++++++++++---
 .../visualizations/public/vis_types/index.ts     |  2 ++
 .../public/vis_types/react_vis_controller.tsx    | 15 +++++----------
 .../public/vis_types/react_vis_type.ts           |  8 +++++++-
 .../public/vis_types/types_service.ts            | 14 ++++++--------
 14 files changed, 61 insertions(+), 39 deletions(-)

diff --git a/src/plugins/input_control_vis/public/input_control_vis_type.ts b/src/plugins/input_control_vis/public/input_control_vis_type.ts
index 9f415f2100004a..782df67f5c58a4 100644
--- a/src/plugins/input_control_vis/public/input_control_vis_type.ts
+++ b/src/plugins/input_control_vis/public/input_control_vis_type.ts
@@ -19,12 +19,15 @@
 
 import { i18n } from '@kbn/i18n';
 
+import { BaseVisTypeOptions } from 'src/plugins/visualizations/public';
 import { createInputControlVisController } from './vis_controller';
 import { getControlsTab } from './components/editor/controls_tab';
 import { OptionsTab } from './components/editor/options_tab';
 import { InputControlVisDependencies } from './plugin';
 
-export function createInputControlVisTypeDefinition(deps: InputControlVisDependencies) {
+export function createInputControlVisTypeDefinition(
+  deps: InputControlVisDependencies
+): BaseVisTypeOptions {
   const InputControlVisController = createInputControlVisController(deps);
   const ControlsTab = getControlsTab(deps);
 
diff --git a/src/plugins/input_control_vis/public/vis_controller.tsx b/src/plugins/input_control_vis/public/vis_controller.tsx
index faea98b7922917..6f35e17866120b 100644
--- a/src/plugins/input_control_vis/public/vis_controller.tsx
+++ b/src/plugins/input_control_vis/public/vis_controller.tsx
@@ -31,12 +31,12 @@ import { RangeControl } from './control/range_control_factory';
 import { ListControl } from './control/list_control_factory';
 import { InputControlVisDependencies } from './plugin';
 import { FilterManager, Filter } from '../../data/public';
-import { VisParams, Vis } from '../../visualizations/public';
+import { VisParams, ExprVis } from '../../visualizations/public';
 
 export const createInputControlVisController = (deps: InputControlVisDependencies) => {
   return class InputControlVisController {
     private I18nContext?: I18nStart['Context'];
-    private isLoaded = false;
+    private _isLoaded = false;
 
     controls: Array<RangeControl | ListControl>;
     queryBarUpdateHandler: () => void;
@@ -45,7 +45,7 @@ export const createInputControlVisController = (deps: InputControlVisDependencie
     timeFilterSubscription: Subscription;
     visParams?: VisParams;
 
-    constructor(public el: Element, public vis: Vis) {
+    constructor(public el: Element, public vis: ExprVis) {
       this.controls = [];
 
       this.queryBarUpdateHandler = this.updateControlsFromKbn.bind(this);
@@ -58,7 +58,7 @@ export const createInputControlVisController = (deps: InputControlVisDependencie
         .getTimeUpdate$()
         .subscribe(() => {
           if (this.visParams?.useTimeFilter) {
-            this.isLoaded = false;
+            this._isLoaded = false;
           }
         });
     }
@@ -68,11 +68,11 @@ export const createInputControlVisController = (deps: InputControlVisDependencie
         const [{ i18n }] = await deps.core.getStartServices();
         this.I18nContext = i18n.Context;
       }
-      if (!this.isLoaded || !isEqual(visParams, this.visParams)) {
+      if (!this._isLoaded || !isEqual(visParams, this.visParams)) {
         this.visParams = visParams;
         this.controls = [];
         this.controls = await this.initControls();
-        this.isLoaded = true;
+        this._isLoaded = true;
       }
       this.drawVis();
     }
diff --git a/src/plugins/vis_type_metric/public/metric_vis_type.ts b/src/plugins/vis_type_metric/public/metric_vis_type.ts
index 6b4d6e151693fb..1c5afd396c2c31 100644
--- a/src/plugins/vis_type_metric/public/metric_vis_type.ts
+++ b/src/plugins/vis_type_metric/public/metric_vis_type.ts
@@ -18,13 +18,14 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { BaseVisTypeOptions } from 'src/plugins/visualizations/public';
 import { MetricVisOptions } from './components/metric_vis_options';
 import { ColorSchemas, colorSchemas, ColorModes } from '../../charts/public';
 import { AggGroupNames } from '../../data/public';
 import { Schemas } from '../../vis_default_editor/public';
 import { toExpressionAst } from './to_ast';
 
-export const createMetricVisTypeDefinition = () => ({
+export const createMetricVisTypeDefinition = (): BaseVisTypeOptions => ({
   name: 'metric',
   title: i18n.translate('visTypeMetric.metricTitle', { defaultMessage: 'Metric' }),
   icon: 'visMetric',
diff --git a/src/plugins/vis_type_table/public/table_vis_controller.test.ts b/src/plugins/vis_type_table/public/table_vis_controller.test.ts
index 56d17c187bd3fc..7535e98d391c6c 100644
--- a/src/plugins/vis_type_table/public/table_vis_controller.test.ts
+++ b/src/plugins/vis_type_table/public/table_vis_controller.test.ts
@@ -121,7 +121,7 @@ describe('Table Vis - Controller', () => {
   function getRangeVis(params?: object) {
     return ({
       type: tableVisTypeDefinition,
-      params: Object.assign({}, tableVisTypeDefinition.visConfig.defaults, params),
+      params: Object.assign({}, tableVisTypeDefinition.visConfig?.defaults, params),
       data: {
         aggs: createAggConfigs(stubIndexPattern, [
           { type: 'count', schema: 'metric' },
diff --git a/src/plugins/vis_type_table/public/table_vis_type.ts b/src/plugins/vis_type_table/public/table_vis_type.ts
index 80d53021b7866d..c1419a48474580 100644
--- a/src/plugins/vis_type_table/public/table_vis_type.ts
+++ b/src/plugins/vis_type_table/public/table_vis_type.ts
@@ -20,7 +20,7 @@ import { CoreSetup, PluginInitializerContext } from 'kibana/public';
 import { i18n } from '@kbn/i18n';
 import { AggGroupNames } from '../../data/public';
 import { Schemas } from '../../vis_default_editor/public';
-import { Vis } from '../../visualizations/public';
+import { BaseVisTypeOptions, Vis } from '../../visualizations/public';
 import { tableVisResponseHandler } from './table_vis_response_handler';
 // @ts-ignore
 import tableVisTemplate from './table_vis.html';
@@ -28,9 +28,11 @@ import { TableOptions } from './components/table_vis_options_lazy';
 import { getTableVisualizationControllerClass } from './vis_controller';
 import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public';
 
-export function getTableVisTypeDefinition(core: CoreSetup, context: PluginInitializerContext) {
+export function getTableVisTypeDefinition(
+  core: CoreSetup,
+  context: PluginInitializerContext
+): BaseVisTypeOptions {
   return {
-    type: 'table',
     name: 'table',
     title: i18n.translate('visTypeTable.tableVisTitle', {
       defaultMessage: 'Data Table',
diff --git a/src/plugins/vis_type_table/public/vis_controller.ts b/src/plugins/vis_type_table/public/vis_controller.ts
index d87812b9f5d694..5e82796e663393 100644
--- a/src/plugins/vis_type_table/public/vis_controller.ts
+++ b/src/plugins/vis_type_table/public/vis_controller.ts
@@ -64,7 +64,7 @@ export function getTableVisualizationControllerClass(
       }
     }
 
-    async render(esResponse: object, visParams: VisParams) {
+    async render(esResponse: object, visParams: VisParams): Promise<void> {
       getKibanaLegacy().loadFontAwesome();
       await this.initLocalAngular();
 
diff --git a/src/plugins/vis_type_vislib/public/vis_controller.tsx b/src/plugins/vis_type_vislib/public/vis_controller.tsx
index 86ef98de045d7c..c422e9f4f3a0af 100644
--- a/src/plugins/vis_type_vislib/public/vis_controller.tsx
+++ b/src/plugins/vis_type_vislib/public/vis_controller.tsx
@@ -68,7 +68,7 @@ export const createVislibVisController = (deps: VisTypeVislibDependencies) => {
       this.container.appendChild(this.legendEl);
     }
 
-    render(esResponse: any, visParams: VisParams) {
+    render(esResponse: any, visParams: VisParams): Promise<void> {
       if (this.vislibVis) {
         this.destroy();
       }
diff --git a/src/plugins/visualizations/public/index.ts b/src/plugins/visualizations/public/index.ts
index 17c292a1b183b7..3e3926bc5c8d84 100644
--- a/src/plugins/visualizations/public/index.ts
+++ b/src/plugins/visualizations/public/index.ts
@@ -36,7 +36,7 @@ export { getSchemas as getVisSchemas } from './legacy/build_pipeline';
 
 /** @public types */
 export { VisualizationsSetup, VisualizationsStart };
-export { VisTypeAlias, VisType } from './vis_types';
+export { VisTypeAlias, VisType, BaseVisTypeOptions, ReactVisTypeOptions } from './vis_types';
 export { VisParams, SerializedVis, SerializedVisData, VisData } from './vis';
 export type VisualizeEmbeddableFactoryContract = PublicContract<VisualizeEmbeddableFactory>;
 export type VisualizeEmbeddableContract = PublicContract<VisualizeEmbeddable>;
diff --git a/src/plugins/visualizations/public/types.ts b/src/plugins/visualizations/public/types.ts
index f47ffbbe921a25..897a8c1e323194 100644
--- a/src/plugins/visualizations/public/types.ts
+++ b/src/plugins/visualizations/public/types.ts
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import { ExpressionAstExpression } from 'src/plugins/expressions';
 import { SavedObject } from '../../../plugins/saved_objects/public';
 import {
   AggConfigOptions,
@@ -24,6 +25,7 @@ import {
   TimefilterContract,
 } from '../../../plugins/data/public';
 import { SerializedVis, Vis, VisParams } from './vis';
+import { ExprVis } from './expressions/vis';
 
 export { Vis, SerializedVis, VisParams };
 
@@ -35,7 +37,7 @@ export interface VisualizationController {
 
 export type VisualizationControllerConstructor = new (
   el: HTMLElement,
-  vis: Vis
+  vis: ExprVis
 ) => VisualizationController;
 
 export interface SavedVisState {
@@ -71,4 +73,7 @@ export interface VisToExpressionAstParams {
   abortSignal?: AbortSignal;
 }
 
-export type VisToExpressionAst = (vis: Vis, params: VisToExpressionAstParams) => string;
+export type VisToExpressionAst = (
+  vis: Vis,
+  params: VisToExpressionAstParams
+) => ExpressionAstExpression;
diff --git a/src/plugins/visualizations/public/vis_types/base_vis_type.ts b/src/plugins/visualizations/public/vis_types/base_vis_type.ts
index fa0bbfc5e250a6..283286648ff16f 100644
--- a/src/plugins/visualizations/public/vis_types/base_vis_type.ts
+++ b/src/plugins/visualizations/public/vis_types/base_vis_type.ts
@@ -22,7 +22,7 @@ import { VisToExpressionAst, VisualizationControllerConstructor } from '../types
 import { TriggerContextMapping } from '../../../ui_actions/public';
 import { Adapters } from '../../../inspector/public';
 
-export interface BaseVisTypeOptions {
+interface CommonBaseVisTypeOptions {
   name: string;
   title: string;
   description?: string;
@@ -31,7 +31,6 @@ export interface BaseVisTypeOptions {
   image?: string;
   stage?: 'experimental' | 'beta' | 'production';
   options?: Record<string, any>;
-  visualization: VisualizationControllerConstructor | undefined;
   visConfig?: Record<string, any>;
   editor?: any;
   editorConfig?: Record<string, any>;
@@ -42,9 +41,20 @@ export interface BaseVisTypeOptions {
   setup?: unknown;
   useCustomNoDataScreen?: boolean;
   inspectorAdapters?: Adapters | (() => Adapters);
-  toExpressionAst?: VisToExpressionAst;
 }
 
+interface ExpressionBaseVisTypeOptions extends CommonBaseVisTypeOptions {
+  toExpressionAst: VisToExpressionAst;
+  visualization?: undefined;
+}
+
+interface VisualizationBaseVisTypeOptions extends CommonBaseVisTypeOptions {
+  toExpressionAst?: undefined;
+  visualization: VisualizationControllerConstructor | undefined;
+}
+
+export type BaseVisTypeOptions = ExpressionBaseVisTypeOptions | VisualizationBaseVisTypeOptions;
+
 export class BaseVisType {
   name: string;
   title: string;
diff --git a/src/plugins/visualizations/public/vis_types/index.ts b/src/plugins/visualizations/public/vis_types/index.ts
index 625c8be414c74d..8f38e335691620 100644
--- a/src/plugins/visualizations/public/vis_types/index.ts
+++ b/src/plugins/visualizations/public/vis_types/index.ts
@@ -18,3 +18,5 @@
  */
 
 export * from './types_service';
+export type { BaseVisTypeOptions } from './base_vis_type';
+export type { ReactVisTypeOptions } from './react_vis_type';
diff --git a/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx b/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx
index 643e6ffcb730b0..ceb6435dce27e5 100644
--- a/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx
+++ b/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx
@@ -19,17 +19,12 @@
 
 import React from 'react';
 import { render, unmountComponentAtNode } from 'react-dom';
-import { Vis, VisualizationController } from '../types';
+import { VisualizationController } from '../types';
 import { getI18n, getUISettings } from '../services';
+import { ExprVis } from '../expressions/vis';
 
 export class ReactVisController implements VisualizationController {
-  private el: HTMLElement;
-  private vis: Vis;
-
-  constructor(element: HTMLElement, vis: Vis) {
-    this.el = element;
-    this.vis = vis;
-  }
+  constructor(private element: HTMLElement, private vis: ExprVis) {}
 
   public render(visData: any, visParams: any): Promise<void> {
     const I18nContext = getI18n().Context;
@@ -51,12 +46,12 @@ export class ReactVisController implements VisualizationController {
             renderComplete={resolve}
           />
         </I18nContext>,
-        this.el
+        this.element
       );
     });
   }
 
   public destroy() {
-    unmountComponentAtNode(this.el);
+    unmountComponentAtNode(this.element);
   }
 }
diff --git a/src/plugins/visualizations/public/vis_types/react_vis_type.ts b/src/plugins/visualizations/public/vis_types/react_vis_type.ts
index 68979abe52a3cb..047d36d8041116 100644
--- a/src/plugins/visualizations/public/vis_types/react_vis_type.ts
+++ b/src/plugins/visualizations/public/vis_types/react_vis_type.ts
@@ -20,8 +20,14 @@
 import { BaseVisType, BaseVisTypeOptions } from './base_vis_type';
 import { ReactVisController } from './react_vis_controller';
 
+export type ReactVisTypeOptions = Omit<BaseVisTypeOptions, 'visualization' | 'toExpressionAst'>;
+
+/**
+ * This class should only be used for visualizations not using the `toExpressionAst` with a custom renderer.
+ * If you implement a custom renderer you should just mount a react component inside this.
+ */
 export class ReactVisType extends BaseVisType {
-  constructor(opts: Omit<BaseVisTypeOptions, 'visualization'>) {
+  constructor(opts: ReactVisTypeOptions) {
     super({
       ...opts,
       visualization: ReactVisController,
diff --git a/src/plugins/visualizations/public/vis_types/types_service.ts b/src/plugins/visualizations/public/vis_types/types_service.ts
index 14c2a9c50ab0eb..157dbd41ce8a21 100644
--- a/src/plugins/visualizations/public/vis_types/types_service.ts
+++ b/src/plugins/visualizations/public/vis_types/types_service.ts
@@ -19,10 +19,8 @@
 
 import { IconType } from '@elastic/eui';
 import { visTypeAliasRegistry, VisTypeAlias } from './vis_type_alias_registry';
-// @ts-ignore
-import { BaseVisType } from './base_vis_type';
-// @ts-ignore
-import { ReactVisType } from './react_vis_type';
+import { BaseVisType, BaseVisTypeOptions } from './base_vis_type';
+import { ReactVisType, ReactVisTypeOptions } from './react_vis_type';
 import { TriggerContextMapping } from '../../../ui_actions/public';
 
 export interface VisType {
@@ -71,17 +69,17 @@ export class TypesService {
     return {
       /**
        * registers a visualization type
-       * @param {VisType} config - visualization type definition
+       * @param config - visualization type definition
        */
-      createBaseVisualization: (config: any) => {
+      createBaseVisualization: (config: BaseVisTypeOptions): void => {
         const vis = new BaseVisType(config);
         registerVisualization(() => vis);
       },
       /**
        * registers a visualization which uses react for rendering
-       * @param {VisType} config - visualization type definition
+       * @param config - visualization type definition
        */
-      createReactVisualization: (config: any) => {
+      createReactVisualization: (config: ReactVisTypeOptions): void => {
         const vis = new ReactVisType(config);
         registerVisualization(() => vis);
       },

From 88df93bed56ea02c9a84f78eac32c9653829c5eb Mon Sep 17 00:00:00 2001
From: Alison Goryachev <alison.goryachev@elastic.co>
Date: Mon, 28 Sep 2020 08:47:05 -0400
Subject: [PATCH 18/22] [Ingest pipelines] Upload indexed document to test a
 pipeline (#77939)

---
 .../helpers/setup_environment.tsx             |   5 +
 .../__jest__/http_requests.helpers.ts         |  12 +
 .../__jest__/test_pipeline.helpers.tsx        |  23 ++
 .../__jest__/test_pipeline.test.tsx           |  86 ++++++-
 .../test_pipeline_flyout.container.tsx        |   7 +
 .../test_pipeline/test_pipeline_flyout.tsx    |  18 +-
 .../add_document_form.tsx                     | 209 ++++++++++++++++++
 .../add_documents_accordion.scss              |   4 +
 .../add_documents_accordion.tsx               | 111 ++++++++++
 .../add_documents_accordion/index.ts          |   7 +
 .../documents_schema.tsx                      |  24 ++
 .../tab_documents.tsx                         | 179 +++++++++------
 .../context/test_pipeline_context.tsx         |   8 +-
 .../use_is_mounted.ts                         |  21 ++
 .../public/application/index.tsx              |   2 +
 .../application/mount_management_section.ts   |   6 +-
 .../public/application/services/api.ts        |   9 +
 .../plugins/ingest_pipelines/public/plugin.ts |   7 +-
 .../ingest_pipelines/public/shared_imports.ts |   2 +
 .../plugins/ingest_pipelines/public/types.ts  |   8 +-
 .../ingest_pipelines/public/url_generator.ts  |   6 +-
 .../server/routes/api/documents.ts            |  56 +++++
 .../server/routes/api/index.ts                |   2 +
 .../ingest_pipelines/server/routes/index.ts   |   2 +
 .../ingest_pipelines/ingest_pipelines.ts      |  62 +++++-
 .../ingest_pipelines/lib/elasticsearch.ts     |  10 +
 26 files changed, 783 insertions(+), 103 deletions(-)
 create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx
 create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss
 create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx
 create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts
 create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts
 create mode 100644 x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts

diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx
index c380032bd9482e..d9a0ac41153895 100644
--- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx
+++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx
@@ -44,6 +44,11 @@ const appServices = {
   api: apiService,
   notifications: notificationServiceMock.createSetupContract(),
   history,
+  urlGenerators: {
+    getUrlGenerator: jest.fn().mockReturnValue({
+      createUrl: jest.fn(),
+    }),
+  },
 };
 
 export const setupEnvironment = () => {
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts
index 541a6853a99b39..c89b07ae0192f4 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts
@@ -21,8 +21,20 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => {
     ]);
   };
 
+  const setFetchDocumentsResponse = (response?: HttpResponse, error?: any) => {
+    const status = error ? error.status || 400 : 200;
+    const body = error ? JSON.stringify(error.body) : JSON.stringify(response);
+
+    server.respondWith('GET', '/api/ingest_pipelines/documents/:index/:id', [
+      status,
+      { 'Content-Type': 'application/json' },
+      body,
+    ]);
+  };
+
   return {
     setSimulatePipelineResponse,
+    setFetchDocumentsResponse,
   };
 };
 
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx
index f4c89d7a1058a5..215ef63d9782ef 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx
@@ -94,6 +94,11 @@ const appServices = {
   notifications: notificationServiceMock.createSetupContract(),
   history,
   uiSettings: {},
+  urlGenerators: {
+    getUrlGenerator: jest.fn().mockReturnValue({
+      createUrl: jest.fn(),
+    }),
+  },
 };
 
 const testBedSetup = registerTestBed<TestSubject>(
@@ -180,6 +185,20 @@ const createActions = (testBed: TestBed<TestSubject>) => {
       });
       component.update();
     },
+
+    async toggleDocumentsAccordion() {
+      await act(async () => {
+        find('addDocumentsAccordion').simulate('click');
+      });
+      component.update();
+    },
+
+    async clickAddDocumentButton() {
+      await act(async () => {
+        find('addDocumentButton').simulate('click');
+      });
+      component.update();
+    },
   };
 };
 
@@ -229,4 +248,8 @@ type TestSubject =
   | 'configurationTab'
   | 'outputTab'
   | 'processorOutputTabContent'
+  | 'addDocumentsAccordion'
+  | 'addDocumentButton'
+  | 'addDocumentError'
+  | 'addDocumentSuccess'
   | string;
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx
index e5118a6e465af0..47f05602799e4e 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx
@@ -141,10 +141,9 @@ describe('Test pipeline', () => {
       const { actions, find, exists } = testBed;
 
       const error = {
-        status: 400,
-        error: 'Bad Request',
-        message:
-          '"[parse_exception] [_source] required property is missing, with { property_name="_source" }"',
+        status: 500,
+        error: 'Internal server error',
+        message: 'Internal server error',
       };
 
       httpRequestsMockHelpers.setSimulatePipelineResponse(undefined, { body: error });
@@ -153,13 +152,90 @@ describe('Test pipeline', () => {
       actions.clickAddDocumentsButton();
 
       // Add invalid sample documents array and run the pipeline
-      actions.addDocumentsJson(JSON.stringify([{}]));
+      actions.addDocumentsJson(
+        JSON.stringify([
+          {
+            _index: 'test',
+            _id: '1',
+            _version: 1,
+            _seq_no: 0,
+            _primary_term: 1,
+            _source: {
+              name: 'John Doe',
+            },
+          },
+        ])
+      );
       await actions.clickRunPipelineButton();
 
       // Verify error rendered
       expect(exists('pipelineExecutionError')).toBe(true);
       expect(find('pipelineExecutionError').text()).toContain(error.message);
     });
+
+    describe('Add indexed documents', () => {
+      test('should successfully add an indexed document', async () => {
+        const { actions, form, exists } = testBed;
+
+        const { _index: index, _id: documentId } = DOCUMENTS[0];
+
+        httpRequestsMockHelpers.setFetchDocumentsResponse(DOCUMENTS[0]);
+
+        // Open flyout
+        actions.clickAddDocumentsButton();
+
+        // Open documents accordion, click run without required fields, and verify error messages
+        await actions.toggleDocumentsAccordion();
+        await actions.clickAddDocumentButton();
+        expect(form.getErrorsMessages()).toEqual([
+          'An index name is required.',
+          'A document ID is required.',
+        ]);
+
+        // Add required fields, and click run
+        form.setInputValue('indexField.input', index);
+        form.setInputValue('idField.input', documentId);
+        await actions.clickAddDocumentButton();
+
+        // Verify request
+        const latestRequest = server.requests[server.requests.length - 1];
+        expect(latestRequest.status).toEqual(200);
+        expect(latestRequest.url).toEqual(`/api/ingest_pipelines/documents/${index}/${documentId}`);
+        // Verify success callout
+        expect(exists('addDocumentSuccess')).toBe(true);
+      });
+
+      test('should surface API errors from the request', async () => {
+        const { actions, form, exists, find } = testBed;
+
+        const nonExistentDoc = {
+          index: 'foo',
+          id: '1',
+        };
+
+        const error = {
+          status: 404,
+          error: 'Not found',
+          message: '[index_not_found_exception] no such index',
+        };
+
+        httpRequestsMockHelpers.setFetchDocumentsResponse(undefined, { body: error });
+
+        // Open flyout
+        actions.clickAddDocumentsButton();
+
+        // Open documents accordion, add required fields, and click run
+        await actions.toggleDocumentsAccordion();
+        form.setInputValue('indexField.input', nonExistentDoc.index);
+        form.setInputValue('idField.input', nonExistentDoc.id);
+        await actions.clickAddDocumentButton();
+
+        // Verify error rendered
+        expect(exists('addDocumentError')).toBe(true);
+        expect(exists('addDocumentSuccess')).toBe(false);
+        expect(find('addDocumentError').text()).toContain(error.message);
+      });
+    });
   });
 
   describe('Processors', () => {
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx
index b49eea5b59ab0f..23dda55db41f81 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx
@@ -12,6 +12,7 @@ import { useTestPipelineContext } from '../../context';
 import { serialize } from '../../serialize';
 import { DeserializeResult } from '../../deserialize';
 import { Document } from '../../types';
+import { useIsMounted } from '../../use_is_mounted';
 import { TestPipelineFlyout as ViewComponent } from './test_pipeline_flyout';
 
 import { TestPipelineFlyoutTab } from './test_pipeline_flyout_tabs';
@@ -34,6 +35,7 @@ export const TestPipelineFlyout: React.FunctionComponent<Props> = ({
   processors,
 }) => {
   const { services } = useKibana();
+  const isMounted = useIsMounted();
 
   const {
     testPipelineData,
@@ -74,6 +76,10 @@ export const TestPipelineFlyout: React.FunctionComponent<Props> = ({
         pipeline: { ...serializedProcessors },
       });
 
+      if (!isMounted.current) {
+        return { isSuccessful: false };
+      }
+
       setIsRunningTest(false);
 
       if (error) {
@@ -123,6 +129,7 @@ export const TestPipelineFlyout: React.FunctionComponent<Props> = ({
       return { isSuccessful: true };
     },
     [
+      isMounted,
       processors,
       services.api,
       services.notifications.toasts,
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx
index 46271a6bce51c2..d4895f88055318 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx
@@ -16,7 +16,7 @@ import {
   EuiCallOut,
 } from '@elastic/eui';
 
-import { Form, FormHook } from '../../../../../shared_imports';
+import { FormHook } from '../../../../../shared_imports';
 import { Document } from '../../types';
 
 import { Tabs, TestPipelineFlyoutTab, OutputTab, DocumentsTab } from './test_pipeline_flyout_tabs';
@@ -71,19 +71,11 @@ export const TestPipelineFlyout: React.FunctionComponent<Props> = ({
   } else {
     // default to "Documents" tab
     tabContent = (
-      <Form
+      <DocumentsTab
         form={form}
-        data-test-subj="testPipelineForm"
-        isInvalid={form.isSubmitted && !form.isValid}
-        onSubmit={validateAndTestPipeline}
-        error={form.getErrors()}
-      >
-        <DocumentsTab
-          validateAndTestPipeline={validateAndTestPipeline}
-          isRunningTest={isRunningTest}
-          isSubmitButtonDisabled={form.isSubmitted && !form.isValid}
-        />
-      </Form>
+        validateAndTestPipeline={validateAndTestPipeline}
+        isRunningTest={isRunningTest}
+      />
     );
   }
 
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx
new file mode 100644
index 00000000000000..340cf1af92300f
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx
@@ -0,0 +1,209 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useState, FunctionComponent } from 'react';
+import { i18n } from '@kbn/i18n';
+import {
+  EuiButton,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiCallOut,
+  EuiSpacer,
+  EuiText,
+  EuiIcon,
+} from '@elastic/eui';
+
+import {
+  getUseField,
+  Field,
+  useKibana,
+  useForm,
+  Form,
+  TextField,
+  fieldValidators,
+  FieldConfig,
+} from '../../../../../../shared_imports';
+import { useIsMounted } from '../../../use_is_mounted';
+import { Document } from '../../../types';
+
+const UseField = getUseField({ component: Field });
+
+const { emptyField } = fieldValidators;
+
+const i18nTexts = {
+  addDocumentButton: i18n.translate(
+    'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentButtonLabel',
+    {
+      defaultMessage: 'Add document',
+    }
+  ),
+  addDocumentErrorMessage: i18n.translate(
+    'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentErrorMessage',
+    {
+      defaultMessage: 'Error adding document',
+    }
+  ),
+  addDocumentSuccessMessage: i18n.translate(
+    'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentSuccessMessage',
+    {
+      defaultMessage: 'Document added',
+    }
+  ),
+  indexField: {
+    fieldLabel: i18n.translate(
+      'xpack.ingestPipelines.pipelineEditor.addDocuments.indexFieldLabel',
+      {
+        defaultMessage: 'Index',
+      }
+    ),
+    validationMessage: i18n.translate(
+      'xpack.ingestPipelines.pipelineEditor.addDocuments.indexRequiredErrorMessage',
+      {
+        defaultMessage: 'An index name is required.',
+      }
+    ),
+  },
+  idField: {
+    fieldLabel: i18n.translate('xpack.ingestPipelines.pipelineEditor.addDocuments.idFieldLabel', {
+      defaultMessage: 'Document ID',
+    }),
+    validationMessage: i18n.translate(
+      'xpack.ingestPipelines.pipelineEditor.addDocuments.idRequiredErrorMessage',
+      {
+        defaultMessage: 'A document ID is required.',
+      }
+    ),
+  },
+};
+
+const fieldsConfig: Record<string, FieldConfig> = {
+  index: {
+    label: i18nTexts.indexField.fieldLabel,
+    validations: [
+      {
+        validator: emptyField(i18nTexts.indexField.validationMessage),
+      },
+    ],
+  },
+  id: {
+    label: i18nTexts.idField.fieldLabel,
+    validations: [
+      {
+        validator: emptyField(i18nTexts.idField.validationMessage),
+      },
+    ],
+  },
+};
+
+interface Props {
+  onAddDocuments: (document: Document) => void;
+}
+
+export const AddDocumentForm: FunctionComponent<Props> = ({ onAddDocuments }) => {
+  const { services } = useKibana();
+  const isMounted = useIsMounted();
+
+  const [isLoadingDocument, setIsLoadingDocument] = useState<boolean>(false);
+  const [documentError, setDocumentError] = useState<Error | undefined>(undefined);
+  const [isDocumentAdded, setIsDocumentAdded] = useState<boolean>(false);
+
+  const { form } = useForm({ defaultValue: { index: '', id: '' } });
+
+  const submitForm = async (e: React.FormEvent) => {
+    const { isValid, data } = await form.submit();
+
+    const { id, index } = data;
+
+    if (isValid) {
+      setIsLoadingDocument(true);
+      setDocumentError(undefined);
+      setIsDocumentAdded(false);
+
+      const { error, data: document } = await services.api.loadDocument(index, id);
+
+      if (!isMounted.current) {
+        return;
+      }
+
+      setIsLoadingDocument(false);
+
+      if (error) {
+        setDocumentError(error);
+        return;
+      }
+
+      setIsDocumentAdded(true);
+      onAddDocuments(document);
+    }
+  };
+
+  return (
+    <Form form={form} onSubmit={submitForm}>
+      {documentError && (
+        <>
+          <EuiCallOut
+            title={i18nTexts.addDocumentErrorMessage}
+            color="danger"
+            iconType="alert"
+            data-test-subj="addDocumentError"
+            size="s"
+          >
+            <p>{documentError.message}</p>
+          </EuiCallOut>
+
+          <EuiSpacer size="m" />
+        </>
+      )}
+
+      <UseField
+        path="index"
+        component={TextField}
+        config={fieldsConfig.index}
+        componentProps={{
+          ['data-test-subj']: 'indexField',
+        }}
+      />
+
+      <UseField
+        path="id"
+        component={TextField}
+        config={fieldsConfig.id}
+        componentProps={{
+          ['data-test-subj']: 'idField',
+        }}
+      />
+
+      <EuiSpacer />
+
+      <EuiFlexGroup>
+        <EuiFlexItem grow={false}>
+          <EuiButton
+            onClick={submitForm}
+            data-test-subj="addDocumentButton"
+            isLoading={isLoadingDocument}
+          >
+            {i18nTexts.addDocumentButton}
+          </EuiButton>
+        </EuiFlexItem>
+
+        {isDocumentAdded && (
+          <EuiFlexItem>
+            <EuiFlexGroup gutterSize="s" alignItems="center">
+              <EuiFlexItem grow={false}>
+                <EuiIcon type="check" color="secondary" />
+              </EuiFlexItem>
+              <EuiFlexItem grow={false}>
+                <EuiText color="secondary" data-test-subj="addDocumentSuccess">
+                  {i18nTexts.addDocumentSuccessMessage}
+                </EuiText>
+              </EuiFlexItem>
+            </EuiFlexGroup>
+          </EuiFlexItem>
+        )}
+      </EuiFlexGroup>
+    </Form>
+  );
+};
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss
new file mode 100644
index 00000000000000..2bf234fab2ece1
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss
@@ -0,0 +1,4 @@
+.addDocumentsAccordion {
+  background-color: $euiColorLightestShade;
+  padding: $euiSizeM;
+}
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx
new file mode 100644
index 00000000000000..88ced6e9e94dd0
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FunctionComponent, useState, useEffect } from 'react';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+import { EuiAccordion, EuiText, EuiSpacer, EuiLink } from '@elastic/eui';
+import { UrlGeneratorsDefinition } from 'src/plugins/share/public';
+
+import { useKibana } from '../../../../../../../shared_imports';
+import { useIsMounted } from '../../../../use_is_mounted';
+import { AddDocumentForm } from '../add_document_form';
+
+import './add_documents_accordion.scss';
+
+const DISCOVER_URL_GENERATOR_ID = 'DISCOVER_APP_URL_GENERATOR';
+
+const i18nTexts = {
+  addDocumentsButton: i18n.translate(
+    'xpack.ingestPipelines.pipelineEditor.addDocumentsAccordion.addDocumentsButtonLabel',
+    {
+      defaultMessage: 'Add documents from index',
+    }
+  ),
+};
+
+interface Props {
+  onAddDocuments: (document: any) => void;
+}
+
+export const AddDocumentsAccordion: FunctionComponent<Props> = ({ onAddDocuments }) => {
+  const { services } = useKibana();
+  const isMounted = useIsMounted();
+  const [discoverLink, setDiscoverLink] = useState<string | undefined>(undefined);
+
+  useEffect(() => {
+    const getDiscoverUrl = async (): Promise<void> => {
+      let isDeprecated: UrlGeneratorsDefinition<typeof DISCOVER_URL_GENERATOR_ID>['isDeprecated'];
+      let createUrl: UrlGeneratorsDefinition<typeof DISCOVER_URL_GENERATOR_ID>['createUrl'];
+
+      // This try/catch may not be necessary once
+      // https://github.com/elastic/kibana/issues/78344 is addressed
+      try {
+        ({ isDeprecated, createUrl } = services.urlGenerators.getUrlGenerator(
+          DISCOVER_URL_GENERATOR_ID
+        ));
+      } catch (e) {
+        // Discover plugin is not enabled
+        setDiscoverLink(undefined);
+        return;
+      }
+
+      if (isDeprecated) {
+        setDiscoverLink(undefined);
+        return;
+      }
+
+      const discoverUrl = await createUrl({ indexPatternId: undefined });
+
+      if (isMounted.current) {
+        setDiscoverLink(discoverUrl);
+      }
+    };
+
+    getDiscoverUrl();
+  }, [isMounted, services.urlGenerators]);
+
+  return (
+    <EuiAccordion
+      id="addDocumentsAccordion"
+      buttonContent={i18nTexts.addDocumentsButton}
+      paddingSize="s"
+      data-test-subj="addDocumentsAccordion"
+    >
+      <div className="addDocumentsAccordion">
+        <EuiText size="s" color="subdued">
+          <p>
+            <FormattedMessage
+              id="xpack.ingestPipelines.pipelineEditor.addDocumentsAccordion.contentDescriptionText"
+              defaultMessage="Provide the index name and document ID of the indexed document to test."
+            />
+            {discoverLink && (
+              <>
+                {' '}
+                <FormattedMessage
+                  id="xpack.ingestPipelines.pipelineEditor.addDocumentsAccordion.discoverLinkDescriptionText"
+                  defaultMessage="To explore your existing data, use {discoverLink}."
+                  values={{
+                    discoverLink: (
+                      <EuiLink href={discoverLink} target="_blank" external>
+                        Discover
+                      </EuiLink>
+                    ),
+                  }}
+                />
+              </>
+            )}
+          </p>
+        </EuiText>
+
+        <EuiSpacer size="m" />
+
+        <AddDocumentForm onAddDocuments={onAddDocuments} />
+      </div>
+    </EuiAccordion>
+  );
+};
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts
new file mode 100644
index 00000000000000..cb00ec640b5a65
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { AddDocumentsAccordion } from './add_documents_accordion';
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx
index e8ac223d56ed93..d0e0596375cb22 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx
@@ -82,6 +82,30 @@ export const documentsSchema: FormSchema = {
           }
         },
       },
+      {
+        validator: ({ value }: ValidationFuncArg<any, any>) => {
+          const parsedJSON = JSON.parse(value);
+
+          const isMissingSourceField = parsedJSON.find((document: { _source?: object }) => {
+            if (!document._source) {
+              return true;
+            }
+
+            return false;
+          });
+
+          if (isMissingSourceField) {
+            return {
+              message: i18n.translate(
+                'xpack.ingestPipelines.testPipelineFlyout.documentsForm.sourceFieldRequiredError',
+                {
+                  defaultMessage: 'Documents require a _source field.',
+                }
+              ),
+            };
+          }
+        },
+      },
     ],
   },
 };
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx
index b2326644340a7d..6fd340054d2a4d 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx
@@ -4,98 +4,131 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React from 'react';
+import React, { FunctionComponent, useCallback } from 'react';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
 
 import { EuiSpacer, EuiText, EuiButton, EuiLink } from '@elastic/eui';
 
-import { getUseField, Field, JsonEditorField, useKibana } from '../../../../../../shared_imports';
+import {
+  getUseField,
+  Field,
+  JsonEditorField,
+  useKibana,
+  useFormData,
+  FormHook,
+  Form,
+} from '../../../../../../shared_imports';
+
+import { AddDocumentsAccordion } from './add_documents_accordion';
 
 const UseField = getUseField({ component: Field });
 
 interface Props {
   validateAndTestPipeline: () => Promise<void>;
   isRunningTest: boolean;
-  isSubmitButtonDisabled: boolean;
+  form: FormHook;
 }
 
-export const DocumentsTab: React.FunctionComponent<Props> = ({
+export const DocumentsTab: FunctionComponent<Props> = ({
   validateAndTestPipeline,
-  isSubmitButtonDisabled,
   isRunningTest,
+  form,
 }) => {
   const { services } = useKibana();
 
+  const [, formatData] = useFormData({ form });
+
+  const onAddDocumentHandler = useCallback(
+    (document) => {
+      const { documents: existingDocuments = [] } = formatData();
+
+      form.reset({ defaultValue: { documents: [...existingDocuments, document] } });
+    },
+    [form, formatData]
+  );
+
   return (
-    <div data-test-subj="documentsTabContent">
-      <EuiText>
-        <p>
-          <FormattedMessage
-            id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.tabDescriptionText"
-            defaultMessage="Provide an array of documents for the pipeline to ingest. {learnMoreLink}"
-            values={{
-              learnMoreLink: (
-                <EuiLink
-                  href={`${services.documentation.getEsDocsBasePath()}/simulate-pipeline-api.html`}
-                  target="_blank"
-                  external
-                >
-                  {i18n.translate(
-                    'xpack.ingestPipelines.testPipelineFlyout.documentsTab.simulateDocumentionLink',
-                    {
-                      defaultMessage: 'Learn more.',
-                    }
-                  )}
-                </EuiLink>
+    <Form
+      form={form}
+      data-test-subj="testPipelineForm"
+      isInvalid={form.isSubmitted && !form.isValid}
+      onSubmit={validateAndTestPipeline}
+      error={form.getErrors()}
+    >
+      <div data-test-subj="documentsTabContent">
+        <EuiText>
+          <p>
+            <FormattedMessage
+              id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.tabDescriptionText"
+              defaultMessage="Provide documents for the pipeline to ingest. {learnMoreLink}"
+              values={{
+                learnMoreLink: (
+                  <EuiLink
+                    href={`${services.documentation.getEsDocsBasePath()}/simulate-pipeline-api.html`}
+                    target="_blank"
+                    external
+                  >
+                    {i18n.translate(
+                      'xpack.ingestPipelines.testPipelineFlyout.documentsTab.simulateDocumentionLink',
+                      {
+                        defaultMessage: 'Learn more.',
+                      }
+                    )}
+                  </EuiLink>
+                ),
+              }}
+            />
+          </p>
+        </EuiText>
+
+        <EuiSpacer size="m" />
+
+        <AddDocumentsAccordion onAddDocuments={onAddDocumentHandler} />
+
+        <EuiSpacer size="l" />
+
+        {/* Documents editor */}
+        <UseField
+          path="documents"
+          component={JsonEditorField}
+          componentProps={{
+            euiCodeEditorProps: {
+              'data-test-subj': 'documentsEditor',
+              height: '300px',
+              'aria-label': i18n.translate(
+                'xpack.ingestPipelines.testPipelineFlyout.documentsTab.editorFieldAriaLabel',
+                {
+                  defaultMessage: 'Documents JSON editor',
+                }
               ),
-            }}
-          />
-        </p>
-      </EuiText>
-
-      <EuiSpacer size="m" />
-
-      {/* Documents editor */}
-      <UseField
-        path="documents"
-        component={JsonEditorField}
-        componentProps={{
-          euiCodeEditorProps: {
-            'data-test-subj': 'documentsEditor',
-            height: '300px',
-            'aria-label': i18n.translate(
-              'xpack.ingestPipelines.testPipelineFlyout.documentsTab.editorFieldAriaLabel',
-              {
-                defaultMessage: 'Documents JSON editor',
-              }
-            ),
-          },
-        }}
-      />
-
-      <EuiSpacer size="m" />
-
-      <EuiButton
-        onClick={validateAndTestPipeline}
-        data-test-subj="runPipelineButton"
-        size="s"
-        isLoading={isRunningTest}
-        disabled={isSubmitButtonDisabled}
-        iconType="play"
-      >
-        {isRunningTest ? (
-          <FormattedMessage
-            id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.runningButtonLabel"
-            defaultMessage="Running"
-          />
-        ) : (
-          <FormattedMessage
-            id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.runButtonLabel"
-            defaultMessage="Run the pipeline"
-          />
-        )}
-      </EuiButton>
-    </div>
+            },
+          }}
+        />
+
+        <EuiSpacer size="m" />
+
+        <EuiButton
+          onClick={validateAndTestPipeline}
+          data-test-subj="runPipelineButton"
+          size="s"
+          isLoading={isRunningTest}
+          disabled={form.isSubmitted && !form.isValid}
+          iconType="play"
+        >
+          {isRunningTest ? (
+            <FormattedMessage
+              id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.runningButtonLabel"
+              defaultMessage="Running"
+            />
+          ) : (
+            <FormattedMessage
+              id="xpack.ingestPipelines.testPipelineFlyout.documentsTab.runButtonLabel"
+              defaultMessage="Run the pipeline"
+            />
+          )}
+        </EuiButton>
+      </div>
+    </Form>
   );
 };
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx
index 9aafeafa10b279..314964f808e448 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx
@@ -15,6 +15,7 @@ import {
 } from '../deserialize';
 import { serialize } from '../serialize';
 import { Document } from '../types';
+import { useIsMounted } from '../use_is_mounted';
 
 export interface TestPipelineData {
   config: {
@@ -127,6 +128,7 @@ export const reducer: Reducer<TestPipelineData, Action> = (state, action) => {
 export const TestPipelineContextProvider = ({ children }: { children: React.ReactNode }) => {
   const [state, dispatch] = useReducer(reducer, DEFAULT_TEST_PIPELINE_CONTEXT.testPipelineData);
   const { services } = useKibana();
+  const isMounted = useIsMounted();
 
   const updateTestOutputPerProcessor = useCallback(
     async (documents: Document[] | undefined, processors: DeserializeResult) => {
@@ -152,6 +154,10 @@ export const TestPipelineContextProvider = ({ children }: { children: React.Reac
         pipeline: { ...serializedProcessorsWithTag },
       });
 
+      if (!isMounted.current) {
+        return;
+      }
+
       if (error) {
         dispatch({
           type: 'updateOutputPerProcessor',
@@ -180,7 +186,7 @@ export const TestPipelineContextProvider = ({ children }: { children: React.Reac
         },
       });
     },
-    [services.api, services.notifications.toasts]
+    [isMounted, services.api, services.notifications.toasts]
   );
 
   return (
diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts
new file mode 100644
index 00000000000000..c0df15e8a7fb79
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useEffect, useRef } from 'react';
+
+export const useIsMounted = () => {
+  const isMounted = useRef(false);
+
+  useEffect(() => {
+    isMounted.current = true;
+
+    return () => {
+      isMounted.current = false;
+    };
+  }, []);
+
+  return isMounted;
+};
diff --git a/x-pack/plugins/ingest_pipelines/public/application/index.tsx b/x-pack/plugins/ingest_pipelines/public/application/index.tsx
index 6ffebd1854b786..0a71babc533153 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/index.tsx
+++ b/x-pack/plugins/ingest_pipelines/public/application/index.tsx
@@ -9,6 +9,7 @@ import React, { ReactNode } from 'react';
 import { render, unmountComponentAtNode } from 'react-dom';
 import { NotificationsSetup, IUiSettingsClient } from 'kibana/public';
 import { ManagementAppMountParams } from 'src/plugins/management/public';
+import { SharePluginStart } from 'src/plugins/share/public';
 import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public';
 
 import { API_BASE_PATH } from '../../common/constants';
@@ -26,6 +27,7 @@ export interface AppServices {
   notifications: NotificationsSetup;
   history: ManagementAppMountParams['history'];
   uiSettings: IUiSettingsClient;
+  urlGenerators: SharePluginStart['urlGenerators'];
 }
 
 export interface CoreServices {
diff --git a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts
index 16ba9f9cd7a12c..f7094a71a7792c 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts
+++ b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts
@@ -6,15 +6,16 @@
 import { CoreSetup } from 'src/core/public';
 import { ManagementAppMountParams } from 'src/plugins/management/public';
 
+import { StartDependencies } from '../types';
 import { documentationService, uiMetricService, apiService, breadcrumbService } from './services';
 import { renderApp } from '.';
 
 export async function mountManagementSection(
-  { http, getStartServices, notifications }: CoreSetup,
+  { http, getStartServices, notifications }: CoreSetup<StartDependencies>,
   params: ManagementAppMountParams
 ) {
   const { element, setBreadcrumbs, history } = params;
-  const [coreStart] = await getStartServices();
+  const [coreStart, depsStart] = await getStartServices();
   const {
     docLinks,
     i18n: { Context: I18nContext },
@@ -31,6 +32,7 @@ export async function mountManagementSection(
     notifications,
     history,
     uiSettings: coreStart.uiSettings,
+    urlGenerators: depsStart.share.urlGenerators,
   };
 
   return renderApp(element, I18nContext, services, { http });
diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts
index 552e0ed0c41b22..2d6ab0477a603d 100644
--- a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts
+++ b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts
@@ -120,6 +120,15 @@ export class ApiService {
 
     return result;
   }
+
+  public async loadDocument(index: string, id: string) {
+    const result = await this.sendRequest({
+      path: `${API_BASE_PATH}/documents/${encodeURIComponent(index)}/${encodeURIComponent(id)}`,
+      method: 'get',
+    });
+
+    return result;
+  }
 }
 
 export const apiService = new ApiService();
diff --git a/x-pack/plugins/ingest_pipelines/public/plugin.ts b/x-pack/plugins/ingest_pipelines/public/plugin.ts
index 6c2f4a08983272..8b60967702742c 100644
--- a/x-pack/plugins/ingest_pipelines/public/plugin.ts
+++ b/x-pack/plugins/ingest_pipelines/public/plugin.ts
@@ -9,11 +9,12 @@ import { CoreSetup, Plugin } from 'src/core/public';
 
 import { PLUGIN_ID } from '../common/constants';
 import { uiMetricService, apiService } from './application/services';
-import { Dependencies } from './types';
+import { SetupDependencies, StartDependencies } from './types';
 import { registerUrlGenerator } from './url_generator';
 
-export class IngestPipelinesPlugin implements Plugin {
-  public setup(coreSetup: CoreSetup, plugins: Dependencies): void {
+export class IngestPipelinesPlugin
+  implements Plugin<void, void, SetupDependencies, StartDependencies> {
+  public setup(coreSetup: CoreSetup<StartDependencies>, plugins: SetupDependencies): void {
     const { management, usageCollection, share } = plugins;
     const { http, getStartServices } = coreSetup;
 
diff --git a/x-pack/plugins/ingest_pipelines/public/shared_imports.ts b/x-pack/plugins/ingest_pipelines/public/shared_imports.ts
index 703b7a90f9356a..13de8a74225abd 100644
--- a/x-pack/plugins/ingest_pipelines/public/shared_imports.ts
+++ b/x-pack/plugins/ingest_pipelines/public/shared_imports.ts
@@ -47,6 +47,7 @@ export {
   getFieldValidityAndErrorMessage,
   ValidationFunc,
   ValidationConfig,
+  useFormData,
 } from '../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib';
 
 export {
@@ -65,6 +66,7 @@ export {
   NumericField,
   SelectField,
   CheckBoxField,
+  TextField,
 } from '../../../../src/plugins/es_ui_shared/static/forms/components';
 
 export {
diff --git a/x-pack/plugins/ingest_pipelines/public/types.ts b/x-pack/plugins/ingest_pipelines/public/types.ts
index e968c87226d074..1638e60e98505b 100644
--- a/x-pack/plugins/ingest_pipelines/public/types.ts
+++ b/x-pack/plugins/ingest_pipelines/public/types.ts
@@ -6,10 +6,14 @@
 
 import { ManagementSetup } from 'src/plugins/management/public';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/public';
-import { SharePluginSetup } from '../../../../src/plugins/share/public';
+import { SharePluginStart, SharePluginSetup } from 'src/plugins/share/public';
 
-export interface Dependencies {
+export interface SetupDependencies {
   management: ManagementSetup;
   usageCollection: UsageCollectionSetup;
   share: SharePluginSetup;
 }
+
+export interface StartDependencies {
+  share: SharePluginStart;
+}
diff --git a/x-pack/plugins/ingest_pipelines/public/url_generator.ts b/x-pack/plugins/ingest_pipelines/public/url_generator.ts
index 043d449a0440a0..c53ff083ea0981 100644
--- a/x-pack/plugins/ingest_pipelines/public/url_generator.ts
+++ b/x-pack/plugins/ingest_pipelines/public/url_generator.ts
@@ -13,7 +13,7 @@ import {
   getEditPath,
   getListPath,
 } from './application/services/navigation';
-import { Dependencies } from './types';
+import { SetupDependencies } from './types';
 import { PLUGIN_ID } from '../common/constants';
 
 export const INGEST_PIPELINES_APP_ULR_GENERATOR = 'INGEST_PIPELINES_APP_URL_GENERATOR';
@@ -83,8 +83,8 @@ export class IngestPipelinesUrlGenerator
 
 export const registerUrlGenerator = (
   coreSetup: CoreSetup,
-  management: Dependencies['management'],
-  share: Dependencies['share']
+  management: SetupDependencies['management'],
+  share: SetupDependencies['share']
 ) => {
   const getAppBasePath = async (absolute = false) => {
     const [coreStart] = await coreSetup.getStartServices();
diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts
new file mode 100644
index 00000000000000..1f19112e069d5f
--- /dev/null
+++ b/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { schema } from '@kbn/config-schema';
+
+import { API_BASE_PATH } from '../../../common/constants';
+import { RouteDependencies } from '../../types';
+
+const paramsSchema = schema.object({
+  index: schema.string(),
+  id: schema.string(),
+});
+
+export const registerDocumentsRoute = ({
+  router,
+  license,
+  lib: { isEsError },
+}: RouteDependencies): void => {
+  router.get(
+    {
+      path: `${API_BASE_PATH}/documents/{index}/{id}`,
+      validate: {
+        params: paramsSchema,
+      },
+    },
+    license.guardApiRoute(async (ctx, req, res) => {
+      const { callAsCurrentUser } = ctx.core.elasticsearch.legacy.client;
+      const { index, id } = req.params;
+
+      try {
+        const document = await callAsCurrentUser('get', { index, id });
+
+        const { _id, _index, _source } = document;
+
+        return res.ok({
+          body: {
+            _id,
+            _index,
+            _source,
+          },
+        });
+      } catch (error) {
+        if (isEsError(error)) {
+          return res.customError({
+            statusCode: error.statusCode,
+            body: error,
+          });
+        }
+
+        return res.internalError({ body: error });
+      }
+    })
+  );
+};
diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts
index 58a4bf56176593..7c0ab19917d1f8 100644
--- a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts
+++ b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts
@@ -15,3 +15,5 @@ export { registerPrivilegesRoute } from './privileges';
 export { registerDeleteRoute } from './delete';
 
 export { registerSimulateRoute } from './simulate';
+
+export { registerDocumentsRoute } from './documents';
diff --git a/x-pack/plugins/ingest_pipelines/server/routes/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/index.ts
index f703a460143f45..5e80be4388b25c 100644
--- a/x-pack/plugins/ingest_pipelines/server/routes/index.ts
+++ b/x-pack/plugins/ingest_pipelines/server/routes/index.ts
@@ -13,6 +13,7 @@ import {
   registerPrivilegesRoute,
   registerDeleteRoute,
   registerSimulateRoute,
+  registerDocumentsRoute,
 } from './api';
 
 export class ApiRoutes {
@@ -23,5 +24,6 @@ export class ApiRoutes {
     registerPrivilegesRoute(dependencies);
     registerDeleteRoute(dependencies);
     registerSimulateRoute(dependencies);
+    registerDocumentsRoute(dependencies);
   }
 }
diff --git a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts
index b3fab42a461144..b80306b0e6d38e 100644
--- a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts
+++ b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts
@@ -14,7 +14,13 @@ const API_BASE_PATH = '/api/ingest_pipelines';
 export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
 
-  const { createPipeline, deletePipeline, cleanupPipelines } = registerEsHelpers(getService);
+  const {
+    createPipeline,
+    deletePipeline,
+    cleanupPipelines,
+    createIndex,
+    deleteIndex,
+  } = registerEsHelpers(getService);
 
   describe('Pipelines', function () {
     after(async () => {
@@ -445,5 +451,59 @@ export default function ({ getService }: FtrProviderContext) {
         expect(body.docs?.length).to.eql(2);
       });
     });
+
+    describe('Fetch documents', () => {
+      const INDEX = 'test_index';
+      const DOCUMENT_ID = '1';
+      const DOCUMENT = {
+        name: 'John Doe',
+      };
+
+      before(async () => {
+        // Create an index with a document that can be used to test GET request
+        try {
+          await createIndex({ id: DOCUMENT_ID, index: INDEX, body: DOCUMENT });
+        } catch (err) {
+          // eslint-disable-next-line no-console
+          console.log('[Setup error] Error creating index');
+          throw err;
+        }
+      });
+
+      after(async () => {
+        // Clean up index created
+        try {
+          await deleteIndex(INDEX);
+        } catch (err) {
+          // eslint-disable-next-line no-console
+          console.log('[Cleanup error] Error deleting index');
+          throw err;
+        }
+      });
+
+      it('should return a document', async () => {
+        const uri = `${API_BASE_PATH}/documents/${INDEX}/${DOCUMENT_ID}`;
+
+        const { body } = await supertest.get(uri).set('kbn-xsrf', 'xxx').expect(200);
+
+        expect(body).to.eql({
+          _index: INDEX,
+          _id: DOCUMENT_ID,
+          _source: DOCUMENT,
+        });
+      });
+
+      it('should return an error if the document does not exist', async () => {
+        const uri = `${API_BASE_PATH}/documents/${INDEX}/2`; // Document 2 does not exist
+
+        const { body } = await supertest.get(uri).set('kbn-xsrf', 'xxx').expect(404);
+
+        expect(body).to.eql({
+          error: 'Not Found',
+          message: 'Not Found',
+          statusCode: 404,
+        });
+      });
+    });
   });
 }
diff --git a/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts b/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts
index 6de91e1154a859..aeed61cb0bf92f 100644
--- a/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts
+++ b/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts
@@ -50,9 +50,19 @@ export const registerEsHelpers = (getService: FtrProviderContext['getService'])
         console.log(`[Cleanup error] Error deleting ES resources: ${err.message}`);
       });
 
+  const createIndex = (index: { index: string; id: string; body: object }) => {
+    return es.index(index);
+  };
+
+  const deleteIndex = (indexName: string) => {
+    return es.indices.delete({ index: indexName });
+  };
+
   return {
     createPipeline,
     deletePipeline,
     cleanupPipelines,
+    createIndex,
+    deleteIndex,
   };
 };

From 966f00ac5956ac32e8aa76c54706cf7901328ba9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cau=C3=AA=20Marcondes?=
 <55978943+cauemarcondes@users.noreply.github.com>
Date: Mon, 28 Sep 2020 14:12:58 +0100
Subject: [PATCH 19/22] [APM] Alerting: Add global option to create all alert
 types (#78151)

* adding alert to service page

* sending on alert per service environment and transaction type

* addressing PR comment

* addressing PR comment

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../AlertingFlyout/index.tsx                  |   4 +-
 .../alerting/ServiceAlertTrigger/index.tsx    |  16 +-
 .../TransactionDurationAlertTrigger/index.tsx |  10 +-
 .../index.tsx                                 |  26 +-
 .../index.tsx                                 |  12 +-
 .../components/alerting/fields.test.tsx       |  61 ++++
 .../apm/public/components/alerting/fields.tsx |  15 +-
 .../alerting/get_alert_capabilities.ts        |  32 ++
 .../Home/alerting_popover_flyout/index.tsx    | 186 ++++++++++
 .../apm/public/components/app/Home/index.tsx  |  25 +-
 .../index.tsx                                 |   6 +-
 .../components/app/ServiceDetails/index.tsx   |  30 +-
 .../register_error_count_alert_type.test.ts   | 197 +++++++++++
 .../alerts/register_error_count_alert_type.ts |  81 ++++-
 ...action_duration_anomaly_alert_type.test.ts | 326 ++++++++++++++++++
 ...transaction_duration_anomaly_alert_type.ts | 117 +++++--
 ..._transaction_error_rate_alert_type.test.ts | 289 ++++++++++++++++
 ...ister_transaction_error_rate_alert_type.ts |  93 ++++-
 .../lib/service_map/get_service_anomalies.ts  |  12 +-
 19 files changed, 1419 insertions(+), 119 deletions(-)
 rename x-pack/plugins/apm/public/components/{app/ServiceDetails/AlertIntegrations => alerting}/AlertingFlyout/index.tsx (86%)
 create mode 100644 x-pack/plugins/apm/public/components/alerting/fields.test.tsx
 create mode 100644 x-pack/plugins/apm/public/components/alerting/get_alert_capabilities.ts
 create mode 100644 x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx
 rename x-pack/plugins/apm/public/components/app/ServiceDetails/{AlertIntegrations => alerting_popover_flyout}/index.tsx (97%)
 create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts
 create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts
 create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts

diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx b/x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx
similarity index 86%
rename from x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx
rename to x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx
index ad3f1696ad5e3d..3bee6b2388264c 100644
--- a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx
@@ -4,8 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import React from 'react';
-import { AlertType } from '../../../../../../common/alert_types';
-import { AlertAdd } from '../../../../../../../triggers_actions_ui/public';
+import { AlertType } from '../../../../common/alert_types';
+import { AlertAdd } from '../../../../../triggers_actions_ui/public';
 
 type AlertAddProps = React.ComponentProps<typeof AlertAdd>;
 
diff --git a/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx
index 86dc7f5a904752..b4d3e8f3ad2411 100644
--- a/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx
@@ -34,11 +34,17 @@ export function ServiceAlertTrigger(props: Props) {
 
   useEffect(() => {
     // we only want to run this on mount to set default values
-    setAlertProperty('name', `${alertTypeName} | ${params.serviceName}`);
-    setAlertProperty('tags', [
-      'apm',
-      `service.name:${params.serviceName}`.toLowerCase(),
-    ]);
+
+    const alertName = params.serviceName
+      ? `${alertTypeName} | ${params.serviceName}`
+      : alertTypeName;
+    setAlertProperty('name', alertName);
+
+    const tags = ['apm'];
+    if (params.serviceName) {
+      tags.push(`service.name:${params.serviceName}`.toLowerCase());
+    }
+    setAlertProperty('tags', tags);
     Object.keys(params).forEach((key) => {
       setAlertParams(key, params[key]);
     });
diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx
index 3ddd623d9e8480..ce98354c94c7eb 100644
--- a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx
@@ -90,16 +90,16 @@ export function TransactionDurationAlertTrigger(props: Props) {
 
   const fields = [
     <ServiceField value={serviceName} />,
-    <EnvironmentField
-      currentValue={params.environment}
-      options={environmentOptions}
-      onChange={(e) => setAlertParams('environment', e.target.value)}
-    />,
     <TransactionTypeField
       currentValue={params.transactionType}
       options={transactionTypes.map((key) => ({ text: key, value: key }))}
       onChange={(e) => setAlertParams('transactionType', e.target.value)}
     />,
+    <EnvironmentField
+      currentValue={params.environment}
+      options={environmentOptions}
+      onChange={(e) => setAlertParams('environment', e.target.value)}
+    />,
     <PopoverExpression
       value={params.aggregationType}
       title={i18n.translate('xpack.apm.transactionDurationAlertTrigger.when', {
diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx
index ca1f55e9d391a7..f910f34d258fdf 100644
--- a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx
@@ -19,10 +19,6 @@ import {
   SelectAnomalySeverity,
 } from './SelectAnomalySeverity';
 import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values';
-import {
-  TRANSACTION_PAGE_LOAD,
-  TRANSACTION_REQUEST,
-} from '../../../../common/transaction_types';
 import {
   EnvironmentField,
   ServiceField,
@@ -32,8 +28,8 @@ import {
 interface Params {
   windowSize: number;
   windowUnit: string;
-  serviceName: string;
-  transactionType: string;
+  serviceName?: string;
+  transactionType?: string;
   environment: string;
   anomalySeverityType:
     | ANOMALY_SEVERITY.CRITICAL
@@ -53,23 +49,17 @@ export function TransactionDurationAnomalyAlertTrigger(props: Props) {
   const { urlParams } = useUrlParams();
   const transactionTypes = useServiceTransactionTypes(urlParams);
   const { serviceName } = useParams<{ serviceName?: string }>();
-  const { start, end } = urlParams;
+  const { start, end, transactionType } = urlParams;
   const { environmentOptions } = useEnvironments({ serviceName, start, end });
-  const supportedTransactionTypes = transactionTypes.filter((transactionType) =>
-    [TRANSACTION_PAGE_LOAD, TRANSACTION_REQUEST].includes(transactionType)
-  );
 
-  if (!supportedTransactionTypes.length || !serviceName) {
+  if (serviceName && !transactionTypes.length) {
     return null;
   }
 
-  // 'page-load' for RUM, 'request' otherwise
-  const transactionType = supportedTransactionTypes[0];
-
   const defaults: Params = {
     windowSize: 15,
     windowUnit: 'm',
-    transactionType,
+    transactionType: transactionType || transactionTypes[0],
     serviceName,
     environment: urlParams.environment || ENVIRONMENT_ALL.value,
     anomalySeverityType: ANOMALY_SEVERITY.CRITICAL,
@@ -82,7 +72,11 @@ export function TransactionDurationAnomalyAlertTrigger(props: Props) {
 
   const fields = [
     <ServiceField value={serviceName} />,
-    <TransactionTypeField currentValue={transactionType} />,
+    <TransactionTypeField
+      currentValue={params.transactionType}
+      options={transactionTypes.map((key) => ({ text: key, value: key }))}
+      onChange={(e) => setAlertParams('transactionType', e.target.value)}
+    />,
     <EnvironmentField
       currentValue={params.environment}
       options={environmentOptions}
diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx
index 4dbf4dc10a9076..a9ad212393ac47 100644
--- a/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx
@@ -43,7 +43,7 @@ export function TransactionErrorRateAlertTrigger(props: Props) {
   const { start, end, transactionType } = urlParams;
   const { environmentOptions } = useEnvironments({ serviceName, start, end });
 
-  if (!transactionTypes.length || !serviceName) {
+  if (serviceName && !transactionTypes.length) {
     return null;
   }
 
@@ -62,16 +62,16 @@ export function TransactionErrorRateAlertTrigger(props: Props) {
 
   const fields = [
     <ServiceField value={serviceName} />,
-    <EnvironmentField
-      currentValue={params.environment}
-      options={environmentOptions}
-      onChange={(e) => setAlertParams('environment', e.target.value)}
-    />,
     <TransactionTypeField
       currentValue={params.transactionType}
       options={transactionTypes.map((key) => ({ text: key, value: key }))}
       onChange={(e) => setAlertParams('transactionType', e.target.value)}
     />,
+    <EnvironmentField
+      currentValue={params.environment}
+      options={environmentOptions}
+      onChange={(e) => setAlertParams('environment', e.target.value)}
+    />,
     <IsAboveField
       value={params.threshold}
       unit="%"
diff --git a/x-pack/plugins/apm/public/components/alerting/fields.test.tsx b/x-pack/plugins/apm/public/components/alerting/fields.test.tsx
new file mode 100644
index 00000000000000..7ffb46d3dda492
--- /dev/null
+++ b/x-pack/plugins/apm/public/components/alerting/fields.test.tsx
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import { ServiceField, TransactionTypeField } from './fields';
+import { act, fireEvent, render } from '@testing-library/react';
+import { expectTextsInDocument } from '../../utils/testHelpers';
+
+describe('alerting fields', () => {
+  describe('Service Fiels', () => {
+    it('renders with value', () => {
+      const component = render(<ServiceField value="foo" />);
+      expectTextsInDocument(component, ['foo']);
+    });
+    it('renders with All when value is not defined', () => {
+      const component = render(<ServiceField />);
+      expectTextsInDocument(component, ['All']);
+    });
+  });
+  describe('Transaction Type Field', () => {
+    it('renders select field when multiple options available', () => {
+      const options = [
+        { text: 'Foo', value: 'foo' },
+        { text: 'Bar', value: 'bar' },
+      ];
+      const { getByText, getByTestId } = render(
+        <TransactionTypeField currentValue="Foo" options={options} />
+      );
+
+      act(() => {
+        fireEvent.click(getByText('Foo'));
+      });
+
+      const selectBar = getByTestId('transactionTypeField');
+      expect(selectBar instanceof HTMLSelectElement).toBeTruthy();
+      const selectOptions = (selectBar as HTMLSelectElement).options;
+      expect(selectOptions.length).toEqual(2);
+      expect(
+        Object.values(selectOptions).map((option) => option.value)
+      ).toEqual(['foo', 'bar']);
+    });
+    it('renders read-only field when single option available', () => {
+      const options = [{ text: 'Bar', value: 'bar' }];
+      const component = render(
+        <TransactionTypeField currentValue="Bar" options={options} />
+      );
+      expectTextsInDocument(component, ['Bar']);
+    });
+    it('renders read-only All option when no option available', () => {
+      const component = render(<TransactionTypeField currentValue="" />);
+      expectTextsInDocument(component, ['All']);
+    });
+
+    it('renders current value when available', () => {
+      const component = render(<TransactionTypeField currentValue="foo" />);
+      expectTextsInDocument(component, ['foo']);
+    });
+  });
+});
diff --git a/x-pack/plugins/apm/public/components/alerting/fields.tsx b/x-pack/plugins/apm/public/components/alerting/fields.tsx
index e145d03671a180..aac64649546cc3 100644
--- a/x-pack/plugins/apm/public/components/alerting/fields.tsx
+++ b/x-pack/plugins/apm/public/components/alerting/fields.tsx
@@ -11,13 +11,17 @@ import { EuiSelectOption } from '@elastic/eui';
 import { getEnvironmentLabel } from '../../../common/environment_filter_values';
 import { PopoverExpression } from './ServiceAlertTrigger/PopoverExpression';
 
+const ALL_OPTION = i18n.translate('xpack.apm.alerting.fields.all_option', {
+  defaultMessage: 'All',
+});
+
 export function ServiceField({ value }: { value?: string }) {
   return (
     <EuiExpression
       description={i18n.translate('xpack.apm.alerting.fields.service', {
         defaultMessage: 'Service',
       })}
-      value={value}
+      value={value || ALL_OPTION}
     />
   );
 }
@@ -53,7 +57,7 @@ export function TransactionTypeField({
   options,
   onChange,
 }: {
-  currentValue: string;
+  currentValue?: string;
   options?: EuiSelectOption[];
   onChange?: (event: React.ChangeEvent<HTMLSelectElement>) => void;
 }) {
@@ -61,13 +65,16 @@ export function TransactionTypeField({
     defaultMessage: 'Type',
   });
 
-  if (!options || options.length === 1) {
-    return <EuiExpression description={label} value={currentValue} />;
+  if (!options || options.length <= 1) {
+    return (
+      <EuiExpression description={label} value={currentValue || ALL_OPTION} />
+    );
   }
 
   return (
     <PopoverExpression value={currentValue} title={label}>
       <EuiSelect
+        data-test-subj="transactionTypeField"
         value={currentValue}
         options={options}
         onChange={onChange}
diff --git a/x-pack/plugins/apm/public/components/alerting/get_alert_capabilities.ts b/x-pack/plugins/apm/public/components/alerting/get_alert_capabilities.ts
new file mode 100644
index 00000000000000..0e03196f0c681c
--- /dev/null
+++ b/x-pack/plugins/apm/public/components/alerting/get_alert_capabilities.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Capabilities } from 'kibana/public';
+import { ApmPluginSetupDeps } from '../../plugin';
+
+export const getAlertingCapabilities = (
+  plugins: ApmPluginSetupDeps,
+  capabilities: Capabilities
+) => {
+  const canReadAlerts = !!capabilities.apm['alerting:show'];
+  const canSaveAlerts = !!capabilities.apm['alerting:save'];
+  const isAlertingPluginEnabled = 'alerts' in plugins;
+  const isAlertingAvailable =
+    isAlertingPluginEnabled && (canReadAlerts || canSaveAlerts);
+  const isMlPluginEnabled = 'ml' in plugins;
+  const canReadAnomalies = !!(
+    isMlPluginEnabled &&
+    capabilities.ml.canAccessML &&
+    capabilities.ml.canGetJobs
+  );
+
+  return {
+    isAlertingAvailable,
+    canReadAlerts,
+    canSaveAlerts,
+    canReadAnomalies,
+  };
+};
diff --git a/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx b/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx
new file mode 100644
index 00000000000000..7e6331c1fa3a8b
--- /dev/null
+++ b/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  EuiButtonEmpty,
+  EuiContextMenu,
+  EuiContextMenuPanelDescriptor,
+  EuiPopover,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import React, { useState } from 'react';
+import { AlertType } from '../../../../../common/alert_types';
+import { useApmPluginContext } from '../../../../hooks/useApmPluginContext';
+import { AlertingFlyout } from '../../../alerting/AlertingFlyout';
+
+const alertLabel = i18n.translate('xpack.apm.home.alertsMenu.alerts', {
+  defaultMessage: 'Alerts',
+});
+const transactionDurationLabel = i18n.translate(
+  'xpack.apm.home.alertsMenu.transactionDuration',
+  { defaultMessage: 'Transaction duration' }
+);
+const transactionErrorRateLabel = i18n.translate(
+  'xpack.apm.home.alertsMenu.transactionErrorRate',
+  { defaultMessage: 'Transaction error rate' }
+);
+const errorCountLabel = i18n.translate('xpack.apm.home.alertsMenu.errorCount', {
+  defaultMessage: 'Error count',
+});
+const createThresholdAlertLabel = i18n.translate(
+  'xpack.apm.home.alertsMenu.createThresholdAlert',
+  { defaultMessage: 'Create threshold alert' }
+);
+const createAnomalyAlertAlertLabel = i18n.translate(
+  'xpack.apm.home.alertsMenu.createAnomalyAlert',
+  { defaultMessage: 'Create anomaly alert' }
+);
+
+const CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID =
+  'create_transaction_duration_panel';
+const CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID =
+  'create_transaction_error_rate_panel';
+const CREATE_ERROR_COUNT_ALERT_PANEL_ID = 'create_error_count_panel';
+
+interface Props {
+  canReadAlerts: boolean;
+  canSaveAlerts: boolean;
+  canReadAnomalies: boolean;
+}
+
+export function AlertingPopoverAndFlyout(props: Props) {
+  const { canSaveAlerts, canReadAlerts, canReadAnomalies } = props;
+
+  const plugin = useApmPluginContext();
+
+  const [popoverOpen, setPopoverOpen] = useState(false);
+
+  const [alertType, setAlertType] = useState<AlertType | null>(null);
+
+  const button = (
+    <EuiButtonEmpty
+      iconType="arrowDown"
+      iconSide="right"
+      onClick={() => setPopoverOpen(true)}
+    >
+      {alertLabel}
+    </EuiButtonEmpty>
+  );
+
+  const panels: EuiContextMenuPanelDescriptor[] = [
+    {
+      id: 0,
+      title: alertLabel,
+      items: [
+        ...(canSaveAlerts
+          ? [
+              {
+                name: transactionDurationLabel,
+                panel: CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID,
+              },
+              {
+                name: transactionErrorRateLabel,
+                panel: CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID,
+              },
+              {
+                name: errorCountLabel,
+                panel: CREATE_ERROR_COUNT_ALERT_PANEL_ID,
+              },
+            ]
+          : []),
+        ...(canReadAlerts
+          ? [
+              {
+                name: i18n.translate(
+                  'xpack.apm.home.alertsMenu.viewActiveAlerts',
+                  { defaultMessage: 'View active alerts' }
+                ),
+                href: plugin.core.http.basePath.prepend(
+                  '/app/management/insightsAndAlerting/triggersActions/alerts'
+                ),
+                icon: 'tableOfContents',
+              },
+            ]
+          : []),
+      ],
+    },
+
+    // transaction duration panel
+    {
+      id: CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID,
+      title: transactionDurationLabel,
+      items: [
+        // anomaly alerts
+        ...(canReadAnomalies
+          ? [
+              {
+                name: createAnomalyAlertAlertLabel,
+                onClick: () => {
+                  setAlertType(AlertType.TransactionDurationAnomaly);
+                  setPopoverOpen(false);
+                },
+              },
+            ]
+          : []),
+      ],
+    },
+
+    // transaction error rate panel
+    {
+      id: CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID,
+      title: transactionErrorRateLabel,
+      items: [
+        // threshold alerts
+        {
+          name: createThresholdAlertLabel,
+          onClick: () => {
+            setAlertType(AlertType.TransactionErrorRate);
+            setPopoverOpen(false);
+          },
+        },
+      ],
+    },
+
+    // error alerts panel
+    {
+      id: CREATE_ERROR_COUNT_ALERT_PANEL_ID,
+      title: errorCountLabel,
+      items: [
+        {
+          name: createThresholdAlertLabel,
+          onClick: () => {
+            setAlertType(AlertType.ErrorCount);
+            setPopoverOpen(false);
+          },
+        },
+      ],
+    },
+  ];
+
+  return (
+    <>
+      <EuiPopover
+        id="integrations-menu"
+        button={button}
+        isOpen={popoverOpen}
+        closePopover={() => setPopoverOpen(false)}
+        panelPaddingSize="none"
+        anchorPosition="downRight"
+      >
+        <EuiContextMenu initialPanelId={0} panels={panels} />
+      </EuiPopover>
+      <AlertingFlyout
+        alertType={alertType}
+        addFlyoutVisible={!!alertType}
+        setAddFlyoutVisibility={(visible) => {
+          if (!visible) {
+            setAlertType(null);
+          }
+        }}
+      />
+    </>
+  );
+}
diff --git a/x-pack/plugins/apm/public/components/app/Home/index.tsx b/x-pack/plugins/apm/public/components/app/Home/index.tsx
index b2f15dbb113412..446f7b978a434e 100644
--- a/x-pack/plugins/apm/public/components/app/Home/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/Home/index.tsx
@@ -15,17 +15,19 @@ import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { $ElementType } from 'utility-types';
 import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
+import { getAlertingCapabilities } from '../../alerting/get_alert_capabilities';
 import { ApmHeader } from '../../shared/ApmHeader';
 import { EuiTabLink } from '../../shared/EuiTabLink';
+import { AnomalyDetectionSetupLink } from '../../shared/Links/apm/AnomalyDetectionSetupLink';
 import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink';
 import { ServiceOverviewLink } from '../../shared/Links/apm/ServiceOverviewLink';
 import { SettingsLink } from '../../shared/Links/apm/SettingsLink';
-import { AnomalyDetectionSetupLink } from '../../shared/Links/apm/AnomalyDetectionSetupLink';
 import { TraceOverviewLink } from '../../shared/Links/apm/TraceOverviewLink';
 import { SetupInstructionsLink } from '../../shared/Links/SetupInstructionsLink';
 import { ServiceMap } from '../ServiceMap';
 import { ServiceOverview } from '../ServiceOverview';
 import { TraceOverview } from '../TraceOverview';
+import { AlertingPopoverAndFlyout } from './alerting_popover_flyout';
 
 function getHomeTabs({
   serviceMapEnabled = true,
@@ -83,13 +85,21 @@ interface Props {
 }
 
 export function Home({ tab }: Props) {
-  const { config, core } = useApmPluginContext();
-  const canAccessML = !!core.application.capabilities.ml?.canAccessML;
+  const { config, core, plugins } = useApmPluginContext();
+  const capabilities = core.application.capabilities;
+  const canAccessML = !!capabilities.ml?.canAccessML;
   const homeTabs = getHomeTabs(config);
   const selectedTab = homeTabs.find(
     (homeTab) => homeTab.name === tab
   ) as $ElementType<typeof homeTabs, number>;
 
+  const {
+    isAlertingAvailable,
+    canReadAlerts,
+    canSaveAlerts,
+    canReadAnomalies,
+  } = getAlertingCapabilities(plugins, core.application.capabilities);
+
   return (
     <div>
       <ApmHeader>
@@ -106,6 +116,15 @@ export function Home({ tab }: Props) {
               </EuiButtonEmpty>
             </SettingsLink>
           </EuiFlexItem>
+          {isAlertingAvailable && (
+            <EuiFlexItem grow={false}>
+              <AlertingPopoverAndFlyout
+                canReadAlerts={canReadAlerts}
+                canSaveAlerts={canSaveAlerts}
+                canReadAnomalies={canReadAnomalies}
+              />
+            </EuiFlexItem>
+          )}
           {canAccessML && (
             <EuiFlexItem grow={false}>
               <AnomalyDetectionSetupLink />
diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx
similarity index 97%
rename from x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx
rename to x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx
index c11bfdeae945be..3a8d24f0a8b02d 100644
--- a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx
@@ -7,14 +7,14 @@
 import {
   EuiButtonEmpty,
   EuiContextMenu,
-  EuiPopover,
   EuiContextMenuPanelDescriptor,
+  EuiPopover,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React, { useState } from 'react';
 import { AlertType } from '../../../../../common/alert_types';
-import { AlertingFlyout } from './AlertingFlyout';
 import { useApmPluginContext } from '../../../../hooks/useApmPluginContext';
+import { AlertingFlyout } from '../../../alerting/AlertingFlyout';
 
 const alertLabel = i18n.translate(
   'xpack.apm.serviceDetails.alertsMenu.alerts',
@@ -53,7 +53,7 @@ interface Props {
   canReadAnomalies: boolean;
 }
 
-export function AlertIntegrations(props: Props) {
+export function AlertingPopoverAndFlyout(props: Props) {
   const { canSaveAlerts, canReadAlerts, canReadAnomalies } = props;
 
   const plugin = useApmPluginContext();
diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx
index 67c4a7c4cde1bb..8825702cafd519 100644
--- a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx
@@ -14,8 +14,9 @@ import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { RouteComponentProps } from 'react-router-dom';
 import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
+import { getAlertingCapabilities } from '../../alerting/get_alert_capabilities';
 import { ApmHeader } from '../../shared/ApmHeader';
-import { AlertIntegrations } from './AlertIntegrations';
+import { AlertingPopoverAndFlyout } from './alerting_popover_flyout';
 import { ServiceDetailTabs } from './ServiceDetailTabs';
 
 interface Props extends RouteComponentProps<{ serviceName: string }> {
@@ -23,20 +24,15 @@ interface Props extends RouteComponentProps<{ serviceName: string }> {
 }
 
 export function ServiceDetails({ match, tab }: Props) {
-  const plugin = useApmPluginContext();
+  const { core, plugins } = useApmPluginContext();
   const { serviceName } = match.params;
-  const capabilities = plugin.core.application.capabilities;
-  const canReadAlerts = !!capabilities.apm['alerting:show'];
-  const canSaveAlerts = !!capabilities.apm['alerting:save'];
-  const isAlertingPluginEnabled = 'alerts' in plugin.plugins;
-  const isAlertingAvailable =
-    isAlertingPluginEnabled && (canReadAlerts || canSaveAlerts);
-  const isMlPluginEnabled = 'ml' in plugin.plugins;
-  const canReadAnomalies = !!(
-    isMlPluginEnabled &&
-    capabilities.ml.canAccessML &&
-    capabilities.ml.canGetJobs
-  );
+
+  const {
+    isAlertingAvailable,
+    canReadAlerts,
+    canSaveAlerts,
+    canReadAnomalies,
+  } = getAlertingCapabilities(plugins, core.application.capabilities);
 
   const ADD_DATA_LABEL = i18n.translate('xpack.apm.addDataButtonLabel', {
     defaultMessage: 'Add data',
@@ -53,7 +49,7 @@ export function ServiceDetails({ match, tab }: Props) {
           </EuiFlexItem>
           {isAlertingAvailable && (
             <EuiFlexItem grow={false}>
-              <AlertIntegrations
+              <AlertingPopoverAndFlyout
                 canReadAlerts={canReadAlerts}
                 canSaveAlerts={canSaveAlerts}
                 canReadAnomalies={canReadAnomalies}
@@ -62,9 +58,7 @@ export function ServiceDetails({ match, tab }: Props) {
           )}
           <EuiFlexItem grow={false}>
             <EuiButtonEmpty
-              href={plugin.core.http.basePath.prepend(
-                '/app/home#/tutorial/apm'
-              )}
+              href={core.http.basePath.prepend('/app/home#/tutorial/apm')}
               size="s"
               color="primary"
               iconType="plusInCircle"
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts
new file mode 100644
index 00000000000000..6a6e2b71370554
--- /dev/null
+++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts
@@ -0,0 +1,197 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Observable } from 'rxjs';
+import * as Rx from 'rxjs';
+import { toArray, map } from 'rxjs/operators';
+
+import { AlertingPlugin } from '../../../../alerts/server';
+import { APMConfig } from '../..';
+
+import { registerErrorCountAlertType } from './register_error_count_alert_type';
+
+type Operator<T1, T2> = (source: Rx.Observable<T1>) => Rx.Observable<T2>;
+const pipeClosure = <T1, T2>(fn: Operator<T1, T2>): Operator<T1, T2> => {
+  return (source: Rx.Observable<T1>) => {
+    return Rx.defer(() => fn(source));
+  };
+};
+const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe(
+  pipeClosure((source$) => {
+    return source$.pipe(map((i) => i));
+  }),
+  toArray()
+) as unknown) as Observable<APMConfig>;
+
+describe('Error count alert', () => {
+  it("doesn't send an alert when error count is less than threshold", async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerErrorCountAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 0,
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(),
+    };
+    const params = { threshold: 1 };
+
+    await alertExecutor!({ services, params });
+    expect(services.alertInstanceFactory).not.toBeCalled();
+  });
+
+  it('sends alerts with service name and environment', async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerErrorCountAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const scheduleActions = jest.fn();
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 2,
+          },
+        },
+        aggregations: {
+          services: {
+            buckets: [
+              {
+                key: 'foo',
+                environments: {
+                  buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }],
+                },
+              },
+              {
+                key: 'bar',
+                environments: {
+                  buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }],
+                },
+              },
+            ],
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+    };
+    const params = { threshold: 1 };
+
+    await alertExecutor!({ services, params });
+    [
+      'apm.error_rate_foo_env-foo',
+      'apm.error_rate_foo_env-foo-2',
+      'apm.error_rate_bar_env-bar',
+      'apm.error_rate_bar_env-bar-2',
+    ].forEach((instanceName) =>
+      expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+    );
+
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      environment: 'env-foo',
+      threshold: 1,
+      triggerValue: 2,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      environment: 'env-foo-2',
+      threshold: 1,
+      triggerValue: 2,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      environment: 'env-bar',
+      threshold: 1,
+      triggerValue: 2,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      environment: 'env-bar-2',
+      threshold: 1,
+      triggerValue: 2,
+    });
+  });
+  it('sends alerts with service name', async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerErrorCountAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const scheduleActions = jest.fn();
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 2,
+          },
+        },
+        aggregations: {
+          services: {
+            buckets: [
+              {
+                key: 'foo',
+              },
+              {
+                key: 'bar',
+              },
+            ],
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+    };
+    const params = { threshold: 1 };
+
+    await alertExecutor!({ services, params });
+    ['apm.error_rate_foo', 'apm.error_rate_bar'].forEach((instanceName) =>
+      expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+    );
+
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      environment: undefined,
+      threshold: 1,
+      triggerValue: 2,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      environment: undefined,
+      threshold: 1,
+      triggerValue: 2,
+    });
+  });
+});
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts
index 5455cd9f6a4951..26e4a5e84b9952 100644
--- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts
+++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts
@@ -5,22 +5,21 @@
  */
 
 import { schema } from '@kbn/config-schema';
+import { isEmpty } from 'lodash';
 import { Observable } from 'rxjs';
 import { take } from 'rxjs/operators';
-import { ProcessorEvent } from '../../../common/processor_event';
-import { getEnvironmentUiFilterES } from '../helpers/convert_ui_filters/get_environment_ui_filter_es';
+import { APMConfig } from '../..';
+import { AlertingPlugin } from '../../../../alerts/server';
 import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types';
-import {
-  ESSearchResponse,
-  ESSearchRequest,
-} from '../../../typings/elasticsearch';
 import {
   PROCESSOR_EVENT,
+  SERVICE_ENVIRONMENT,
   SERVICE_NAME,
 } from '../../../common/elasticsearch_fieldnames';
-import { AlertingPlugin } from '../../../../alerts/server';
+import { ProcessorEvent } from '../../../common/processor_event';
+import { ESSearchResponse } from '../../../typings/elasticsearch';
+import { getEnvironmentUiFilterES } from '../helpers/convert_ui_filters/get_environment_ui_filter_es';
 import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
-import { APMConfig } from '../..';
 import { apmActionVariables } from './action_variables';
 
 interface RegisterAlertParams {
@@ -32,7 +31,7 @@ const paramsSchema = schema.object({
   windowSize: schema.number(),
   windowUnit: schema.string(),
   threshold: schema.number(),
-  serviceName: schema.string(),
+  serviceName: schema.maybe(schema.string()),
   environment: schema.string(),
 });
 
@@ -83,30 +82,74 @@ export function registerErrorCountAlertType({
                   },
                 },
                 { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } },
-                { term: { [SERVICE_NAME]: alertParams.serviceName } },
+                ...(alertParams.serviceName
+                  ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }]
+                  : []),
                 ...getEnvironmentUiFilterES(alertParams.environment),
               ],
             },
           },
+          aggs: {
+            services: {
+              terms: {
+                field: SERVICE_NAME,
+                size: 50,
+              },
+              aggs: {
+                environments: {
+                  terms: {
+                    field: SERVICE_ENVIRONMENT,
+                  },
+                },
+              },
+            },
+          },
         },
       };
 
       const response: ESSearchResponse<
         unknown,
-        ESSearchRequest
+        typeof searchParams
       > = await services.callCluster('search', searchParams);
 
       const errorCount = response.hits.total.value;
 
       if (errorCount > alertParams.threshold) {
-        const alertInstance = services.alertInstanceFactory(
-          AlertType.ErrorCount
-        );
-        alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
-          serviceName: alertParams.serviceName,
-          environment: alertParams.environment,
-          threshold: alertParams.threshold,
-          triggerValue: errorCount,
+        function scheduleAction({
+          serviceName,
+          environment,
+        }: {
+          serviceName: string;
+          environment?: string;
+        }) {
+          const alertInstanceName = [
+            AlertType.ErrorCount,
+            serviceName,
+            environment,
+          ]
+            .filter((name) => name)
+            .join('_');
+
+          const alertInstance = services.alertInstanceFactory(
+            alertInstanceName
+          );
+          alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
+            serviceName,
+            environment,
+            threshold: alertParams.threshold,
+            triggerValue: errorCount,
+          });
+        }
+        response.aggregations?.services.buckets.forEach((serviceBucket) => {
+          const serviceName = serviceBucket.key as string;
+          if (isEmpty(serviceBucket.environments?.buckets)) {
+            scheduleAction({ serviceName });
+          } else {
+            serviceBucket.environments.buckets.forEach((envBucket) => {
+              const environment = envBucket.key as string;
+              scheduleAction({ serviceName, environment });
+            });
+          }
         });
       }
     },
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts
new file mode 100644
index 00000000000000..6e97262dd77bb4
--- /dev/null
+++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts
@@ -0,0 +1,326 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Observable } from 'rxjs';
+import * as Rx from 'rxjs';
+import { toArray, map } from 'rxjs/operators';
+import { AlertingPlugin } from '../../../../alerts/server';
+import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type';
+import { APMConfig } from '../..';
+import { ANOMALY_SEVERITY } from '../../../../ml/common';
+import { Job, MlPluginSetup } from '../../../../ml/server';
+import * as GetServiceAnomalies from '../service_map/get_service_anomalies';
+
+type Operator<T1, T2> = (source: Rx.Observable<T1>) => Rx.Observable<T2>;
+const pipeClosure = <T1, T2>(fn: Operator<T1, T2>): Operator<T1, T2> => {
+  return (source: Rx.Observable<T1>) => {
+    return Rx.defer(() => fn(source));
+  };
+};
+const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe(
+  pipeClosure((source$) => {
+    return source$.pipe(map((i) => i));
+  }),
+  toArray()
+) as unknown) as Observable<APMConfig>;
+
+describe('Transaction duration anomaly alert', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+  describe("doesn't send alert", () => {
+    it('ml is not defined', async () => {
+      let alertExecutor: any;
+      const alerts = {
+        registerType: ({ executor }) => {
+          alertExecutor = executor;
+        },
+      } as AlertingPlugin['setup'];
+
+      registerTransactionDurationAnomalyAlertType({
+        alerts,
+        ml: undefined,
+        config$: mockedConfig$,
+      });
+      expect(alertExecutor).toBeDefined();
+
+      const services = {
+        callCluster: jest.fn(),
+        alertInstanceFactory: jest.fn(),
+      };
+      const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR };
+
+      await alertExecutor!({ services, params });
+      expect(services.callCluster).not.toHaveBeenCalled();
+      expect(services.alertInstanceFactory).not.toHaveBeenCalled();
+    });
+
+    it('ml jobs are not available', async () => {
+      jest
+        .spyOn(GetServiceAnomalies, 'getMLJobs')
+        .mockReturnValue(Promise.resolve([]));
+
+      let alertExecutor: any;
+
+      const alerts = {
+        registerType: ({ executor }) => {
+          alertExecutor = executor;
+        },
+      } as AlertingPlugin['setup'];
+
+      const ml = ({
+        mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }),
+        anomalyDetectorsProvider: jest.fn(),
+      } as unknown) as MlPluginSetup;
+
+      registerTransactionDurationAnomalyAlertType({
+        alerts,
+        ml,
+        config$: mockedConfig$,
+      });
+      expect(alertExecutor).toBeDefined();
+
+      const services = {
+        callCluster: jest.fn(),
+        alertInstanceFactory: jest.fn(),
+      };
+      const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR };
+
+      await alertExecutor!({ services, params });
+      expect(services.callCluster).not.toHaveBeenCalled();
+      expect(services.alertInstanceFactory).not.toHaveBeenCalled();
+    });
+
+    it('anomaly is less than threshold', async () => {
+      jest
+        .spyOn(GetServiceAnomalies, 'getMLJobs')
+        .mockReturnValue(
+          Promise.resolve([{ job_id: '1' }, { job_id: '2' }] as Job[])
+        );
+
+      let alertExecutor: any;
+
+      const alerts = {
+        registerType: ({ executor }) => {
+          alertExecutor = executor;
+        },
+      } as AlertingPlugin['setup'];
+
+      const ml = ({
+        mlSystemProvider: () => ({
+          mlAnomalySearch: () => ({
+            hits: { total: { value: 0 } },
+          }),
+        }),
+        anomalyDetectorsProvider: jest.fn(),
+      } as unknown) as MlPluginSetup;
+
+      registerTransactionDurationAnomalyAlertType({
+        alerts,
+        ml,
+        config$: mockedConfig$,
+      });
+      expect(alertExecutor).toBeDefined();
+
+      const services = {
+        callCluster: jest.fn(),
+        alertInstanceFactory: jest.fn(),
+      };
+      const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR };
+
+      await alertExecutor!({ services, params });
+      expect(services.callCluster).not.toHaveBeenCalled();
+      expect(services.alertInstanceFactory).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('sends alert', () => {
+    it('with service name, environment and transaction type', async () => {
+      jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue(
+        Promise.resolve([
+          {
+            job_id: '1',
+            custom_settings: {
+              job_tags: {
+                environment: 'production',
+              },
+            },
+          } as unknown,
+          {
+            job_id: '2',
+            custom_settings: {
+              job_tags: {
+                environment: 'production',
+              },
+            },
+          } as unknown,
+        ] as Job[])
+      );
+
+      let alertExecutor: any;
+
+      const alerts = {
+        registerType: ({ executor }) => {
+          alertExecutor = executor;
+        },
+      } as AlertingPlugin['setup'];
+
+      const ml = ({
+        mlSystemProvider: () => ({
+          mlAnomalySearch: () => ({
+            hits: { total: { value: 2 } },
+            aggregations: {
+              services: {
+                buckets: [
+                  {
+                    key: 'foo',
+                    transaction_types: {
+                      buckets: [{ key: 'type-foo' }],
+                    },
+                  },
+                  {
+                    key: 'bar',
+                    transaction_types: {
+                      buckets: [{ key: 'type-bar' }],
+                    },
+                  },
+                ],
+              },
+            },
+          }),
+        }),
+        anomalyDetectorsProvider: jest.fn(),
+      } as unknown) as MlPluginSetup;
+
+      registerTransactionDurationAnomalyAlertType({
+        alerts,
+        ml,
+        config$: mockedConfig$,
+      });
+      expect(alertExecutor).toBeDefined();
+
+      const scheduleActions = jest.fn();
+      const services = {
+        callCluster: jest.fn(),
+        alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+      };
+      const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR };
+
+      await alertExecutor!({ services, params });
+
+      await alertExecutor!({ services, params });
+      [
+        'apm.transaction_duration_anomaly_foo_production_type-foo',
+        'apm.transaction_duration_anomaly_bar_production_type-bar',
+      ].forEach((instanceName) =>
+        expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+      );
+
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'foo',
+        transactionType: 'type-foo',
+        environment: 'production',
+      });
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'bar',
+        transactionType: 'type-bar',
+        environment: 'production',
+      });
+    });
+
+    it('with service name', async () => {
+      jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue(
+        Promise.resolve([
+          {
+            job_id: '1',
+            custom_settings: {
+              job_tags: {
+                environment: 'production',
+              },
+            },
+          } as unknown,
+          {
+            job_id: '2',
+            custom_settings: {
+              job_tags: {
+                environment: 'testing',
+              },
+            },
+          } as unknown,
+        ] as Job[])
+      );
+
+      let alertExecutor: any;
+
+      const alerts = {
+        registerType: ({ executor }) => {
+          alertExecutor = executor;
+        },
+      } as AlertingPlugin['setup'];
+
+      const ml = ({
+        mlSystemProvider: () => ({
+          mlAnomalySearch: () => ({
+            hits: { total: { value: 2 } },
+            aggregations: {
+              services: {
+                buckets: [{ key: 'foo' }, { key: 'bar' }],
+              },
+            },
+          }),
+        }),
+        anomalyDetectorsProvider: jest.fn(),
+      } as unknown) as MlPluginSetup;
+
+      registerTransactionDurationAnomalyAlertType({
+        alerts,
+        ml,
+        config$: mockedConfig$,
+      });
+      expect(alertExecutor).toBeDefined();
+
+      const scheduleActions = jest.fn();
+      const services = {
+        callCluster: jest.fn(),
+        alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+      };
+      const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR };
+
+      await alertExecutor!({ services, params });
+
+      await alertExecutor!({ services, params });
+      [
+        'apm.transaction_duration_anomaly_foo_production',
+        'apm.transaction_duration_anomaly_foo_testing',
+        'apm.transaction_duration_anomaly_bar_production',
+        'apm.transaction_duration_anomaly_bar_testing',
+      ].forEach((instanceName) =>
+        expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+      );
+
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'foo',
+        transactionType: undefined,
+        environment: 'production',
+      });
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'bar',
+        transactionType: undefined,
+        environment: 'production',
+      });
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'foo',
+        transactionType: undefined,
+        environment: 'testing',
+      });
+      expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+        serviceName: 'bar',
+        transactionType: undefined,
+        environment: 'testing',
+      });
+    });
+  });
+});
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts
index 61cd79b6727351..36b7964e8128d9 100644
--- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts
+++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts
@@ -6,6 +6,7 @@
 
 import { schema } from '@kbn/config-schema';
 import { Observable } from 'rxjs';
+import { isEmpty } from 'lodash';
 import { ANOMALY_SEVERITY } from '../../../../ml/common';
 import { KibanaRequest } from '../../../../../../src/core/server';
 import {
@@ -16,7 +17,7 @@ import {
 import { AlertingPlugin } from '../../../../alerts/server';
 import { APMConfig } from '../..';
 import { MlPluginSetup } from '../../../../ml/server';
-import { getMLJobIds } from '../service_map/get_service_anomalies';
+import { getMLJobs } from '../service_map/get_service_anomalies';
 import { apmActionVariables } from './action_variables';
 
 interface RegisterAlertParams {
@@ -26,8 +27,8 @@ interface RegisterAlertParams {
 }
 
 const paramsSchema = schema.object({
-  serviceName: schema.string(),
-  transactionType: schema.string(),
+  serviceName: schema.maybe(schema.string()),
+  transactionType: schema.maybe(schema.string()),
   windowSize: schema.number(),
   windowUnit: schema.string(),
   environment: schema.string(),
@@ -72,10 +73,7 @@ export function registerTransactionDurationAnomalyAlertType({
       const { mlAnomalySearch } = ml.mlSystemProvider(request);
       const anomalyDetectors = ml.anomalyDetectorsProvider(request);
 
-      const mlJobIds = await getMLJobIds(
-        anomalyDetectors,
-        alertParams.environment
-      );
+      const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment);
 
       const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find(
         (option) => option.type === alertParams.anomalySeverityType
@@ -89,19 +87,19 @@ export function registerTransactionDurationAnomalyAlertType({
 
       const threshold = selectedOption.threshold;
 
-      if (mlJobIds.length === 0) {
+      if (mlJobs.length === 0) {
         return {};
       }
 
       const anomalySearchParams = {
+        terminateAfter: 1,
         body: {
-          terminateAfter: 1,
           size: 0,
           query: {
             bool: {
               filter: [
                 { term: { result_type: 'record' } },
-                { terms: { job_id: mlJobIds } },
+                { terms: { job_id: mlJobs.map((job) => job.job_id) } },
                 {
                   range: {
                     timestamp: {
@@ -110,11 +108,24 @@ export function registerTransactionDurationAnomalyAlertType({
                     },
                   },
                 },
-                {
-                  term: {
-                    partition_field_value: alertParams.serviceName,
-                  },
-                },
+                ...(alertParams.serviceName
+                  ? [
+                      {
+                        term: {
+                          partition_field_value: alertParams.serviceName,
+                        },
+                      },
+                    ]
+                  : []),
+                ...(alertParams.transactionType
+                  ? [
+                      {
+                        term: {
+                          by_field_value: alertParams.transactionType,
+                        },
+                      },
+                    ]
+                  : []),
                 {
                   range: {
                     record_score: {
@@ -125,22 +136,82 @@ export function registerTransactionDurationAnomalyAlertType({
               ],
             },
           },
+          aggs: {
+            services: {
+              terms: {
+                field: 'partition_field_value',
+                size: 50,
+              },
+              aggs: {
+                transaction_types: {
+                  terms: {
+                    field: 'by_field_value',
+                  },
+                },
+              },
+            },
+          },
         },
       };
 
       const response = ((await mlAnomalySearch(
         anomalySearchParams
-      )) as unknown) as { hits: { total: { value: number } } };
+      )) as unknown) as {
+        hits: { total: { value: number } };
+        aggregations?: {
+          services: {
+            buckets: Array<{
+              key: string;
+              transaction_types: { buckets: Array<{ key: string }> };
+            }>;
+          };
+        };
+      };
+
       const hitCount = response.hits.total.value;
 
       if (hitCount > 0) {
-        const alertInstance = services.alertInstanceFactory(
-          AlertType.TransactionDurationAnomaly
-        );
-        alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
-          serviceName: alertParams.serviceName,
-          transactionType: alertParams.transactionType,
-          environment: alertParams.environment,
+        function scheduleAction({
+          serviceName,
+          environment,
+          transactionType,
+        }: {
+          serviceName: string;
+          environment?: string;
+          transactionType?: string;
+        }) {
+          const alertInstanceName = [
+            AlertType.TransactionDurationAnomaly,
+            serviceName,
+            environment,
+            transactionType,
+          ]
+            .filter((name) => name)
+            .join('_');
+
+          const alertInstance = services.alertInstanceFactory(
+            alertInstanceName
+          );
+          alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
+            serviceName,
+            environment,
+            transactionType,
+          });
+        }
+
+        mlJobs.map((job) => {
+          const environment = job.custom_settings?.job_tags?.environment;
+          response.aggregations?.services.buckets.forEach((serviceBucket) => {
+            const serviceName = serviceBucket.key as string;
+            if (isEmpty(serviceBucket.transaction_types?.buckets)) {
+              scheduleAction({ serviceName, environment });
+            } else {
+              serviceBucket.transaction_types?.buckets.forEach((typeBucket) => {
+                const transactionType = typeBucket.key as string;
+                scheduleAction({ serviceName, environment, transactionType });
+              });
+            }
+          });
         });
       }
     },
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts
new file mode 100644
index 00000000000000..90db48f84b5d95
--- /dev/null
+++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts
@@ -0,0 +1,289 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Observable } from 'rxjs';
+import * as Rx from 'rxjs';
+import { toArray, map } from 'rxjs/operators';
+import { AlertingPlugin } from '../../../../alerts/server';
+import { APMConfig } from '../..';
+import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type';
+
+type Operator<T1, T2> = (source: Rx.Observable<T1>) => Rx.Observable<T2>;
+const pipeClosure = <T1, T2>(fn: Operator<T1, T2>): Operator<T1, T2> => {
+  return (source: Rx.Observable<T1>) => {
+    return Rx.defer(() => fn(source));
+  };
+};
+const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe(
+  pipeClosure((source$) => {
+    return source$.pipe(map((i) => i));
+  }),
+  toArray()
+) as unknown) as Observable<APMConfig>;
+
+describe('Transaction error rate alert', () => {
+  it("doesn't send an alert when rate is less than threshold", async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerTransactionErrorRateAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 0,
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(),
+    };
+    const params = { threshold: 1 };
+
+    await alertExecutor!({ services, params });
+    expect(services.alertInstanceFactory).not.toBeCalled();
+  });
+
+  it('sends alerts with service name, transaction type and environment', async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerTransactionErrorRateAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const scheduleActions = jest.fn();
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 4,
+          },
+        },
+        aggregations: {
+          erroneous_transactions: {
+            doc_count: 2,
+          },
+          services: {
+            buckets: [
+              {
+                key: 'foo',
+                transaction_types: {
+                  buckets: [
+                    {
+                      key: 'type-foo',
+                      environments: {
+                        buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }],
+                      },
+                    },
+                  ],
+                },
+              },
+              {
+                key: 'bar',
+                transaction_types: {
+                  buckets: [
+                    {
+                      key: 'type-bar',
+                      environments: {
+                        buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }],
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+    };
+    const params = { threshold: 10 };
+
+    await alertExecutor!({ services, params });
+    [
+      'apm.transaction_error_rate_foo_type-foo_env-foo',
+      'apm.transaction_error_rate_foo_type-foo_env-foo-2',
+      'apm.transaction_error_rate_bar_type-bar_env-bar',
+      'apm.transaction_error_rate_bar_type-bar_env-bar-2',
+    ].forEach((instanceName) =>
+      expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+    );
+
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      transactionType: 'type-foo',
+      environment: 'env-foo',
+      threshold: 10,
+      triggerValue: 50,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      transactionType: 'type-foo',
+      environment: 'env-foo-2',
+      threshold: 10,
+      triggerValue: 50,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      transactionType: 'type-bar',
+      environment: 'env-bar',
+      threshold: 10,
+      triggerValue: 50,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      transactionType: 'type-bar',
+      environment: 'env-bar-2',
+      threshold: 10,
+      triggerValue: 50,
+    });
+  });
+  it('sends alerts with service name and transaction type', async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerTransactionErrorRateAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const scheduleActions = jest.fn();
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 4,
+          },
+        },
+        aggregations: {
+          erroneous_transactions: {
+            doc_count: 2,
+          },
+          services: {
+            buckets: [
+              {
+                key: 'foo',
+                transaction_types: {
+                  buckets: [{ key: 'type-foo' }],
+                },
+              },
+              {
+                key: 'bar',
+                transaction_types: {
+                  buckets: [{ key: 'type-bar' }],
+                },
+              },
+            ],
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+    };
+    const params = { threshold: 10 };
+
+    await alertExecutor!({ services, params });
+    [
+      'apm.transaction_error_rate_foo_type-foo',
+      'apm.transaction_error_rate_bar_type-bar',
+    ].forEach((instanceName) =>
+      expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+    );
+
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      transactionType: 'type-foo',
+      environment: undefined,
+      threshold: 10,
+      triggerValue: 50,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      transactionType: 'type-bar',
+      environment: undefined,
+      threshold: 10,
+      triggerValue: 50,
+    });
+  });
+
+  it('sends alerts with service name', async () => {
+    let alertExecutor: any;
+    const alerts = {
+      registerType: ({ executor }) => {
+        alertExecutor = executor;
+      },
+    } as AlertingPlugin['setup'];
+
+    registerTransactionErrorRateAlertType({
+      alerts,
+      config$: mockedConfig$,
+    });
+    expect(alertExecutor).toBeDefined();
+
+    const scheduleActions = jest.fn();
+    const services = {
+      callCluster: jest.fn(() => ({
+        hits: {
+          total: {
+            value: 4,
+          },
+        },
+        aggregations: {
+          erroneous_transactions: {
+            doc_count: 2,
+          },
+          services: {
+            buckets: [{ key: 'foo' }, { key: 'bar' }],
+          },
+        },
+      })),
+      alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
+    };
+    const params = { threshold: 10 };
+
+    await alertExecutor!({ services, params });
+    [
+      'apm.transaction_error_rate_foo',
+      'apm.transaction_error_rate_bar',
+    ].forEach((instanceName) =>
+      expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName)
+    );
+
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'foo',
+      transactionType: undefined,
+      environment: undefined,
+      threshold: 10,
+      triggerValue: 50,
+    });
+    expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
+      serviceName: 'bar',
+      transactionType: undefined,
+      environment: undefined,
+      threshold: 10,
+      triggerValue: 50,
+    });
+  });
+});
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts
index a6ed40fc15ec63..e14360029e5dd5 100644
--- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts
+++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts
@@ -7,6 +7,7 @@
 import { schema } from '@kbn/config-schema';
 import { Observable } from 'rxjs';
 import { take } from 'rxjs/operators';
+import { isEmpty } from 'lodash';
 import { ProcessorEvent } from '../../../common/processor_event';
 import { EventOutcome } from '../../../common/event_outcome';
 import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types';
@@ -16,6 +17,7 @@ import {
   SERVICE_NAME,
   TRANSACTION_TYPE,
   EVENT_OUTCOME,
+  SERVICE_ENVIRONMENT,
 } from '../../../common/elasticsearch_fieldnames';
 import { AlertingPlugin } from '../../../../alerts/server';
 import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
@@ -32,8 +34,8 @@ const paramsSchema = schema.object({
   windowSize: schema.number(),
   windowUnit: schema.string(),
   threshold: schema.number(),
-  transactionType: schema.string(),
-  serviceName: schema.string(),
+  transactionType: schema.maybe(schema.string()),
+  serviceName: schema.maybe(schema.string()),
   environment: schema.string(),
 });
 
@@ -84,8 +86,18 @@ export function registerTransactionErrorRateAlertType({
                   },
                 },
                 { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } },
-                { term: { [SERVICE_NAME]: alertParams.serviceName } },
-                { term: { [TRANSACTION_TYPE]: alertParams.transactionType } },
+                ...(alertParams.serviceName
+                  ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }]
+                  : []),
+                ...(alertParams.transactionType
+                  ? [
+                      {
+                        term: {
+                          [TRANSACTION_TYPE]: alertParams.transactionType,
+                        },
+                      },
+                    ]
+                  : []),
                 ...getEnvironmentUiFilterES(alertParams.environment),
               ],
             },
@@ -94,6 +106,24 @@ export function registerTransactionErrorRateAlertType({
             erroneous_transactions: {
               filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } },
             },
+            services: {
+              terms: {
+                field: SERVICE_NAME,
+                size: 50,
+              },
+              aggs: {
+                transaction_types: {
+                  terms: { field: TRANSACTION_TYPE },
+                  aggs: {
+                    environments: {
+                      terms: {
+                        field: SERVICE_ENVIRONMENT,
+                      },
+                    },
+                  },
+                },
+              },
+            },
           },
         },
       };
@@ -114,16 +144,53 @@ export function registerTransactionErrorRateAlertType({
         (errornousTransactionsCount / totalTransactionCount) * 100;
 
       if (transactionErrorRate > alertParams.threshold) {
-        const alertInstance = services.alertInstanceFactory(
-          AlertType.TransactionErrorRate
-        );
+        function scheduleAction({
+          serviceName,
+          environment,
+          transactionType,
+        }: {
+          serviceName: string;
+          environment?: string;
+          transactionType?: string;
+        }) {
+          const alertInstanceName = [
+            AlertType.TransactionErrorRate,
+            serviceName,
+            transactionType,
+            environment,
+          ]
+            .filter((name) => name)
+            .join('_');
+
+          const alertInstance = services.alertInstanceFactory(
+            alertInstanceName
+          );
+          alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
+            serviceName,
+            transactionType,
+            environment,
+            threshold: alertParams.threshold,
+            triggerValue: transactionErrorRate,
+          });
+        }
 
-        alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
-          serviceName: alertParams.serviceName,
-          transactionType: alertParams.transactionType,
-          environment: alertParams.environment,
-          threshold: alertParams.threshold,
-          triggerValue: transactionErrorRate,
+        response.aggregations?.services.buckets.forEach((serviceBucket) => {
+          const serviceName = serviceBucket.key as string;
+          if (isEmpty(serviceBucket.transaction_types?.buckets)) {
+            scheduleAction({ serviceName });
+          } else {
+            serviceBucket.transaction_types.buckets.forEach((typeBucket) => {
+              const transactionType = typeBucket.key as string;
+              if (isEmpty(typeBucket.environments?.buckets)) {
+                scheduleAction({ serviceName, transactionType });
+              } else {
+                typeBucket.environments.buckets.forEach((envBucket) => {
+                  const environment = envBucket.key as string;
+                  scheduleAction({ serviceName, transactionType, environment });
+                });
+              }
+            });
+          }
         });
       }
     },
diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
index 44c0c96142096a..895fc70d76af1d 100644
--- a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
+++ b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
@@ -180,7 +180,7 @@ function transformResponseToServiceAnomalies(
   return serviceAnomaliesMap;
 }
 
-export async function getMLJobIds(
+export async function getMLJobs(
   anomalyDetectors: ReturnType<MlPluginSetup['anomalyDetectorsProvider']>,
   environment?: string
 ) {
@@ -198,7 +198,15 @@ export async function getMLJobIds(
     if (!matchingMLJob) {
       return [];
     }
-    return [matchingMLJob.job_id];
+    return [matchingMLJob];
   }
+  return mlJobs;
+}
+
+export async function getMLJobIds(
+  anomalyDetectors: ReturnType<MlPluginSetup['anomalyDetectorsProvider']>,
+  environment?: string
+) {
+  const mlJobs = await getMLJobs(anomalyDetectors, environment);
   return mlJobs.map((job) => job.job_id);
 }

From 53d49381c84df5f7eeae5d912917324d3c4333b4 Mon Sep 17 00:00:00 2001
From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com>
Date: Mon, 28 Sep 2020 16:14:30 +0300
Subject: [PATCH 20/22] Implement tagcloud renderer (#77910)

* Implement toExpressionAst for tagcloud

* Implement tagcloud vis renderer

* Use resize observer

* Use common no data message

* Update build_pipeline.test

* Update tag cloud tests

* Revert "Use common no data message"

This reverts commit fddf019575f4e22aced1ef1f262d8b499d0e8da7.

* Update interpreter functional tests

* Add tests for toExpressionAst fn

* Use throttled chart update

* Update renderer

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../__snapshots__/tag_cloud_fn.test.ts.snap   |  31 ++-
 .../public/__snapshots__/to_ast.test.ts.snap  | 171 ++++++++++++++
 .../vis_type_tagcloud/public/_tag_cloud.scss  |  14 --
 .../public/components/label.js                |   2 +-
 .../public/components/tag_cloud.scss          |  26 +++
 .../public/components/tag_cloud_chart.tsx     |  84 +++++++
 .../components/tag_cloud_visualization.js     | 216 +++++++++---------
 .../tag_cloud_visualization.test.js           |  77 ++-----
 .../vis_type_tagcloud/public/index.scss       |   8 -
 .../vis_type_tagcloud/public/plugin.ts        |  10 +-
 .../vis_type_tagcloud/public/tag_cloud_fn.ts  |  26 +--
 .../public/tag_cloud_type.ts                  |  14 +-
 .../public/tag_cloud_vis_renderer.tsx         |  54 +++++
 .../vis_type_tagcloud/public/to_ast.test.ts   |  84 +++++++
 .../vis_type_tagcloud/public/to_ast.ts        |  60 +++++
 src/plugins/visualizations/public/index.ts    |   2 +-
 .../__snapshots__/build_pipeline.test.ts.snap |   6 -
 .../public/legacy/build_pipeline.test.ts      |  22 --
 .../public/legacy/build_pipeline.ts           |  60 +----
 src/plugins/visualizations/public/vis.ts      |   4 +-
 .../snapshots/baseline/partial_test_1.json    |   2 +-
 .../snapshots/baseline/tagcloud_all_data.json |   2 +-
 .../snapshots/baseline/tagcloud_fontsize.json |   2 +-
 .../baseline/tagcloud_metric_data.json        |   2 +-
 .../snapshots/baseline/tagcloud_options.json  |   2 +-
 .../snapshots/session/partial_test_1.json     |   2 +-
 .../snapshots/session/tagcloud_all_data.json  |   2 +-
 .../snapshots/session/tagcloud_fontsize.json  |   2 +-
 .../session/tagcloud_metric_data.json         |   2 +-
 .../snapshots/session/tagcloud_options.json   |   2 +-
 30 files changed, 674 insertions(+), 317 deletions(-)
 create mode 100644 src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap
 delete mode 100644 src/plugins/vis_type_tagcloud/public/_tag_cloud.scss
 create mode 100644 src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss
 create mode 100644 src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx
 delete mode 100644 src/plugins/vis_type_tagcloud/public/index.scss
 create mode 100644 src/plugins/vis_type_tagcloud/public/tag_cloud_vis_renderer.tsx
 create mode 100644 src/plugins/vis_type_tagcloud/public/to_ast.test.ts
 create mode 100644 src/plugins/vis_type_tagcloud/public/to_ast.ts

diff --git a/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap b/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap
index 8e28be33515f7b..debc7ab27c632b 100644
--- a/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap
+++ b/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap
@@ -2,25 +2,9 @@
 
 exports[`interpreter/functions#tagcloud returns an object with the correct structure 1`] = `
 Object {
-  "as": "visualization",
+  "as": "tagloud_vis",
   "type": "render",
   "value": Object {
-    "params": Object {
-      "listenOnChange": true,
-    },
-    "visConfig": Object {
-      "maxFontSize": 72,
-      "metric": Object {
-        "accessor": 0,
-        "format": Object {
-          "id": "number",
-        },
-      },
-      "minFontSize": 18,
-      "orientation": "single",
-      "scale": "linear",
-      "showLabel": true,
-    },
     "visData": Object {
       "columns": Array [
         Object {
@@ -35,6 +19,19 @@ Object {
       ],
       "type": "kibana_datatable",
     },
+    "visParams": Object {
+      "maxFontSize": 72,
+      "metric": Object {
+        "accessor": 0,
+        "format": Object {
+          "id": "number",
+        },
+      },
+      "minFontSize": 18,
+      "orientation": "single",
+      "scale": "linear",
+      "showLabel": true,
+    },
     "visType": "tagcloud",
   },
 }
diff --git a/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap b/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap
new file mode 100644
index 00000000000000..d64bdfb1f46f94
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap
@@ -0,0 +1,171 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`tagcloud vis toExpressionAst function should match snapshot params fulfilled 1`] = `
+Object {
+  "chain": Array [
+    Object {
+      "arguments": Object {
+        "aggConfigs": Array [
+          "[]",
+        ],
+        "includeFormatHints": Array [
+          false,
+        ],
+        "index": Array [
+          "123",
+        ],
+        "metricsAtAllLevels": Array [
+          false,
+        ],
+        "partialRows": Array [
+          false,
+        ],
+      },
+      "function": "esaggs",
+      "type": "function",
+    },
+    Object {
+      "arguments": Object {
+        "bucket": Array [
+          Object {
+            "chain": Array [
+              Object {
+                "arguments": Object {
+                  "accessor": Array [
+                    0,
+                  ],
+                  "format": Array [
+                    "terms",
+                  ],
+                  "formatParams": Array [
+                    "{\\"id\\":\\"string\\",\\"otherBucketLabel\\":\\"Other\\",\\"missingBucketLabel\\":\\"Missing\\"}",
+                  ],
+                },
+                "function": "visdimension",
+                "type": "function",
+              },
+            ],
+            "type": "expression",
+          },
+        ],
+        "maxFontSize": Array [
+          15,
+        ],
+        "metric": Array [
+          Object {
+            "chain": Array [
+              Object {
+                "arguments": Object {
+                  "accessor": Array [
+                    1,
+                  ],
+                  "format": Array [
+                    "number",
+                  ],
+                },
+                "function": "visdimension",
+                "type": "function",
+              },
+            ],
+            "type": "expression",
+          },
+        ],
+        "minFontSize": Array [
+          5,
+        ],
+        "orientation": Array [
+          "single",
+        ],
+        "scale": Array [
+          "linear",
+        ],
+        "showLabel": Array [
+          true,
+        ],
+      },
+      "function": "tagcloud",
+      "type": "function",
+    },
+  ],
+  "type": "expression",
+}
+`;
+
+exports[`tagcloud vis toExpressionAst function should match snapshot without params 1`] = `
+Object {
+  "chain": Array [
+    Object {
+      "arguments": Object {
+        "aggConfigs": Array [
+          "[]",
+        ],
+        "includeFormatHints": Array [
+          false,
+        ],
+        "index": Array [
+          "123",
+        ],
+        "metricsAtAllLevels": Array [
+          false,
+        ],
+        "partialRows": Array [
+          false,
+        ],
+      },
+      "function": "esaggs",
+      "type": "function",
+    },
+    Object {
+      "arguments": Object {
+        "bucket": Array [
+          Object {
+            "chain": Array [
+              Object {
+                "arguments": Object {
+                  "accessor": Array [
+                    0,
+                  ],
+                  "format": Array [
+                    "terms",
+                  ],
+                  "formatParams": Array [
+                    "{\\"id\\":\\"string\\",\\"otherBucketLabel\\":\\"Other\\",\\"missingBucketLabel\\":\\"Missing\\"}",
+                  ],
+                },
+                "function": "visdimension",
+                "type": "function",
+              },
+            ],
+            "type": "expression",
+          },
+        ],
+        "metric": Array [
+          Object {
+            "chain": Array [
+              Object {
+                "arguments": Object {
+                  "accessor": Array [
+                    1,
+                  ],
+                  "format": Array [
+                    "number",
+                  ],
+                },
+                "function": "visdimension",
+                "type": "function",
+              },
+            ],
+            "type": "expression",
+          },
+        ],
+        "showLabel": Array [
+          false,
+        ],
+      },
+      "function": "tagcloud",
+      "type": "function",
+    },
+  ],
+  "type": "expression",
+}
+`;
diff --git a/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss b/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss
deleted file mode 100644
index 08901bebc03497..00000000000000
--- a/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss
+++ /dev/null
@@ -1,14 +0,0 @@
-.tgcVis {
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  height: 100%;
-  overflow: hidden;
-}
-
-.tgcVisLabel {
-  width: 100%;
-  text-align: center;
-  font-weight: $euiFontWeightBold;
-}
diff --git a/src/plugins/vis_type_tagcloud/public/components/label.js b/src/plugins/vis_type_tagcloud/public/components/label.js
index 168ec4b270fde8..88b3c2f851138e 100644
--- a/src/plugins/vis_type_tagcloud/public/components/label.js
+++ b/src/plugins/vis_type_tagcloud/public/components/label.js
@@ -28,7 +28,7 @@ export class Label extends Component {
   render() {
     return (
       <div
-        className="tgcVisLabel"
+        className="tgcChart__label"
         style={{ display: this.state.shouldShowLabel ? 'block' : 'none' }}
       >
         {this.state.label}
diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss b/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss
new file mode 100644
index 00000000000000..37867f1ed1c178
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss
@@ -0,0 +1,26 @@
+// Prefix all styles with "tgc" to avoid conflicts.
+// Examples
+//   tgcChart
+//   tgcChart__legend
+//   tgcChart__legend--small
+//   tgcChart__legend-isLoading
+
+.tgcChart__container, .tgcChart__wrapper {
+  flex: 1 1 0;
+  display: flex;
+}
+
+.tgcChart {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  overflow: hidden;
+}
+
+.tgcChart__label {
+  width: 100%;
+  text-align: center;
+  font-weight: $euiFontWeightBold;
+}
diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx
new file mode 100644
index 00000000000000..18a09ec9f49698
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useEffect, useMemo, useRef } from 'react';
+import { EuiResizeObserver } from '@elastic/eui';
+import { throttle } from 'lodash';
+
+import { TagCloudVisDependencies } from '../plugin';
+import { TagCloudVisRenderValue } from '../tag_cloud_fn';
+// @ts-ignore
+import { TagCloudVisualization } from './tag_cloud_visualization';
+
+import './tag_cloud.scss';
+
+type TagCloudChartProps = TagCloudVisDependencies &
+  TagCloudVisRenderValue & {
+    fireEvent: (event: any) => void;
+    renderComplete: () => void;
+  };
+
+export const TagCloudChart = ({
+  colors,
+  visData,
+  visParams,
+  fireEvent,
+  renderComplete,
+}: TagCloudChartProps) => {
+  const chartDiv = useRef<HTMLDivElement>(null);
+  const visController = useRef<any>(null);
+
+  useEffect(() => {
+    visController.current = new TagCloudVisualization(chartDiv.current, colors, fireEvent);
+    return () => {
+      visController.current.destroy();
+      visController.current = null;
+    };
+  }, [colors, fireEvent]);
+
+  useEffect(() => {
+    if (visController.current) {
+      visController.current.render(visData, visParams).then(renderComplete);
+    }
+  }, [visData, visParams, renderComplete]);
+
+  const updateChartSize = useMemo(
+    () =>
+      throttle(() => {
+        if (visController.current) {
+          visController.current.render().then(renderComplete);
+        }
+      }, 300),
+    [renderComplete]
+  );
+
+  return (
+    <EuiResizeObserver onResize={updateChartSize}>
+      {(resizeRef) => (
+        <div className="tgcChart__wrapper" ref={resizeRef}>
+          <div className="tgcChart__container" ref={chartDiv} />
+        </div>
+      )}
+    </EuiResizeObserver>
+  );
+};
+
+// default export required for React.Lazy
+// eslint-disable-next-line import/no-default-export
+export { TagCloudChart as default };
diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js
index e43b3bdc747ab8..5ec22d2c6a4d9e 100644
--- a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js
+++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js
@@ -32,126 +32,138 @@ import d3 from 'd3';
 
 const MAX_TAG_COUNT = 200;
 
-export function createTagCloudVisualization({ colors }) {
-  const colorScale = d3.scale.ordinal().range(colors.seedColors);
-  return class TagCloudVisualization {
-    constructor(node, vis) {
-      this._containerNode = node;
-
-      const cloudRelativeContainer = document.createElement('div');
-      cloudRelativeContainer.classList.add('tgcVis');
-      cloudRelativeContainer.setAttribute('style', 'position: relative');
-      const cloudContainer = document.createElement('div');
-      cloudContainer.classList.add('tgcVis');
-      cloudContainer.setAttribute('data-test-subj', 'tagCloudVisualization');
-      this._containerNode.classList.add('visChart--vertical');
-      cloudRelativeContainer.appendChild(cloudContainer);
-      this._containerNode.appendChild(cloudRelativeContainer);
-
-      this._vis = vis;
-      this._truncated = false;
-      this._tagCloud = new TagCloud(cloudContainer, colorScale);
-      this._tagCloud.on('select', (event) => {
-        if (!this._visParams.bucket) {
-          return;
-        }
-        this._vis.API.events.filter({
-          table: event.meta.data,
-          column: 0,
-          row: event.meta.rowIndex,
-        });
-      });
-      this._renderComplete$ = Rx.fromEvent(this._tagCloud, 'renderComplete');
-
-      this._feedbackNode = document.createElement('div');
-      this._containerNode.appendChild(this._feedbackNode);
-      this._feedbackMessage = React.createRef();
-      render(
-        <I18nProvider>
-          <FeedbackMessage ref={this._feedbackMessage} />
-        </I18nProvider>,
-        this._feedbackNode
-      );
-
-      this._labelNode = document.createElement('div');
-      this._containerNode.appendChild(this._labelNode);
-      this._label = React.createRef();
-      render(<Label ref={this._label} />, this._labelNode);
-    }
+export class TagCloudVisualization {
+  constructor(node, colors, fireEvent) {
+    this._containerNode = node;
+
+    const cloudRelativeContainer = document.createElement('div');
+    cloudRelativeContainer.classList.add('tgcChart');
+    cloudRelativeContainer.setAttribute('style', 'position: relative');
+    const cloudContainer = document.createElement('div');
+    cloudContainer.classList.add('tgcChart');
+    cloudContainer.setAttribute('data-test-subj', 'tagCloudVisualization');
+    this._containerNode.classList.add('visChart--vertical');
+    cloudRelativeContainer.appendChild(cloudContainer);
+    this._containerNode.appendChild(cloudRelativeContainer);
+
+    this._truncated = false;
+    this._tagCloud = new TagCloud(cloudContainer, d3.scale.ordinal().range(colors.seedColors));
+    this._tagCloud.on('select', (event) => {
+      if (!this._visParams.bucket) {
+        return;
+      }
 
-    async render(data, visParams) {
+      fireEvent({
+        name: 'filterBucket',
+        data: {
+          data: [
+            {
+              table: event.meta.data,
+              column: 0,
+              row: event.meta.rowIndex,
+            },
+          ],
+        },
+      });
+    });
+    this._renderComplete$ = Rx.fromEvent(this._tagCloud, 'renderComplete');
+
+    this._feedbackNode = document.createElement('div');
+    this._containerNode.appendChild(this._feedbackNode);
+    this._feedbackMessage = React.createRef();
+    render(
+      <I18nProvider>
+        <FeedbackMessage ref={this._feedbackMessage} />
+      </I18nProvider>,
+      this._feedbackNode
+    );
+
+    this._labelNode = document.createElement('div');
+    this._containerNode.appendChild(this._labelNode);
+    this._label = React.createRef();
+    render(<Label ref={this._label} />, this._labelNode);
+  }
+
+  async render(data, visParams) {
+    if (data && visParams) {
       this._updateParams(visParams);
       this._updateData(data);
-      this._resize();
+    }
 
-      await this._renderComplete$.pipe(take(1)).toPromise();
+    this._resize();
 
-      if (data.columns.length !== 2) {
-        this._feedbackMessage.current.setState({
-          shouldShowTruncate: false,
-          shouldShowIncomplete: false,
-        });
-        return;
-      }
+    await this._renderComplete$.pipe(take(1)).toPromise();
 
+    if (data && data.columns.length !== 2 && this._feedbackMessage.current) {
+      this._feedbackMessage.current.setState({
+        shouldShowTruncate: false,
+        shouldShowIncomplete: false,
+      });
+      return;
+    }
+
+    if (data && this._label.current) {
       this._label.current.setState({
         label: `${data.columns[0].name} - ${data.columns[1].name}`,
         shouldShowLabel: visParams.showLabel,
       });
+    }
+
+    if (this._feedbackMessage.current) {
       this._feedbackMessage.current.setState({
         shouldShowTruncate: this._truncated,
         shouldShowIncomplete: this._tagCloud.getStatus() === TagCloud.STATUS.INCOMPLETE,
       });
     }
-
-    destroy() {
-      this._tagCloud.destroy();
-      unmountComponentAtNode(this._feedbackNode);
-      unmountComponentAtNode(this._labelNode);
+  }
+
+  destroy() {
+    this._tagCloud.destroy();
+    unmountComponentAtNode(this._feedbackNode);
+    unmountComponentAtNode(this._labelNode);
+  }
+
+  _updateData(data) {
+    if (!data || !data.rows.length) {
+      this._tagCloud.setData([]);
+      return;
     }
 
-    _updateData(data) {
-      if (!data || !data.rows.length) {
-        this._tagCloud.setData([]);
-        return;
-      }
-
-      const bucket = this._visParams.bucket;
-      const metric = this._visParams.metric;
-      const bucketFormatter = bucket ? getFormatService().deserialize(bucket.format) : null;
-      const tagColumn = bucket ? data.columns[bucket.accessor].id : -1;
-      const metricColumn = data.columns[metric.accessor].id;
-      const tags = data.rows.map((row, rowIndex) => {
-        const tag = row[tagColumn] === undefined ? 'all' : row[tagColumn];
-        const metric = row[metricColumn];
-        return {
-          displayText: bucketFormatter ? bucketFormatter.convert(tag, 'text') : tag,
-          rawText: tag,
-          value: metric,
-          meta: {
-            data: data,
-            rowIndex: rowIndex,
-          },
-        };
-      });
-
-      if (tags.length > MAX_TAG_COUNT) {
-        tags.length = MAX_TAG_COUNT;
-        this._truncated = true;
-      } else {
-        this._truncated = false;
-      }
-
-      this._tagCloud.setData(tags);
+    const bucket = this._visParams.bucket;
+    const metric = this._visParams.metric;
+    const bucketFormatter = bucket ? getFormatService().deserialize(bucket.format) : null;
+    const tagColumn = bucket ? data.columns[bucket.accessor].id : -1;
+    const metricColumn = data.columns[metric.accessor].id;
+    const tags = data.rows.map((row, rowIndex) => {
+      const tag = row[tagColumn] === undefined ? 'all' : row[tagColumn];
+      const metric = row[metricColumn];
+      return {
+        displayText: bucketFormatter ? bucketFormatter.convert(tag, 'text') : tag,
+        rawText: tag,
+        value: metric,
+        meta: {
+          data: data,
+          rowIndex: rowIndex,
+        },
+      };
+    });
+
+    if (tags.length > MAX_TAG_COUNT) {
+      tags.length = MAX_TAG_COUNT;
+      this._truncated = true;
+    } else {
+      this._truncated = false;
     }
 
-    _updateParams(visParams) {
-      this._visParams = visParams;
-      this._tagCloud.setOptions(visParams);
-    }
+    this._tagCloud.setData(tags);
+  }
 
-    _resize() {
-      this._tagCloud.resize();
-    }
-  };
+  _updateParams(visParams) {
+    this._visParams = visParams;
+    this._tagCloud.setOptions(visParams);
+  }
+
+  _resize() {
+    this._tagCloud.resize();
+  }
 }
diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.test.js b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.test.js
index afdc2737827091..7da77ce4785cb2 100644
--- a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.test.js
+++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.test.js
@@ -19,8 +19,7 @@
 
 import 'jest-canvas-mock';
 
-import { createTagCloudVisTypeDefinition } from '../tag_cloud_type';
-import { createTagCloudVisualization } from './tag_cloud_visualization';
+import { TagCloudVisualization } from './tag_cloud_visualization';
 import { setFormatService } from '../services';
 import { dataPluginMock } from '../../../data/public/mocks';
 import { setHTMLElementOffset, setSVGElementGetBBox } from '../../../../test_utils/public';
@@ -29,7 +28,7 @@ const seedColors = ['#00a69b', '#57c17b', '#6f87d8', '#663db8', '#bc52bc', '#9e3
 
 describe('TagCloudVisualizationTest', () => {
   let domNode;
-  let vis;
+  let visParams;
   let SVGElementGetBBoxSpyInstance;
   let HTMLElementOffsetMockInstance;
 
@@ -52,11 +51,6 @@ describe('TagCloudVisualizationTest', () => {
       { 'col-0': 'BR', 'col-1': 3 },
     ],
   };
-  const TagCloudVisualization = createTagCloudVisualization({
-    colors: {
-      seedColors,
-    },
-  });
 
   const originTransformSVGElement = window.SVGElement.prototype.transform;
 
@@ -82,67 +76,44 @@ describe('TagCloudVisualizationTest', () => {
 
   describe('TagCloudVisualization - basics', () => {
     beforeEach(async () => {
-      const visType = createTagCloudVisTypeDefinition({ colors: seedColors });
       setupDOM(512, 512);
 
-      vis = {
-        type: visType,
-        params: {
-          bucket: { accessor: 0, format: {} },
-          metric: { accessor: 0, format: {} },
-          scale: 'linear',
-          orientation: 'single',
-        },
-        data: {},
+      visParams = {
+        bucket: { accessor: 0, format: {} },
+        metric: { accessor: 0, format: {} },
+        scale: 'linear',
+        orientation: 'single',
       };
     });
 
     test('simple draw', async () => {
-      const tagcloudVisualization = new TagCloudVisualization(domNode, vis);
-
-      await tagcloudVisualization.render(dummyTableGroup, vis.params, {
-        resize: false,
-        params: true,
-        aggs: true,
-        data: true,
-        uiState: false,
+      const tagcloudVisualization = new TagCloudVisualization(domNode, {
+        seedColors,
       });
 
+      await tagcloudVisualization.render(dummyTableGroup, visParams);
+
       const svgNode = domNode.querySelector('svg');
       expect(svgNode.outerHTML).toMatchSnapshot();
     });
 
     test('with resize', async () => {
-      const tagcloudVisualization = new TagCloudVisualization(domNode, vis);
-      await tagcloudVisualization.render(dummyTableGroup, vis.params, {
-        resize: false,
-        params: true,
-        aggs: true,
-        data: true,
-        uiState: false,
+      const tagcloudVisualization = new TagCloudVisualization(domNode, {
+        seedColors,
       });
+      await tagcloudVisualization.render(dummyTableGroup, visParams);
 
-      await tagcloudVisualization.render(dummyTableGroup, vis.params, {
-        resize: true,
-        params: false,
-        aggs: false,
-        data: false,
-        uiState: false,
-      });
+      await tagcloudVisualization.render(dummyTableGroup, visParams);
 
       const svgNode = domNode.querySelector('svg');
       expect(svgNode.outerHTML).toMatchSnapshot();
     });
 
     test('with param change', async function () {
-      const tagcloudVisualization = new TagCloudVisualization(domNode, vis);
-      await tagcloudVisualization.render(dummyTableGroup, vis.params, {
-        resize: false,
-        params: true,
-        aggs: true,
-        data: true,
-        uiState: false,
+      const tagcloudVisualization = new TagCloudVisualization(domNode, {
+        seedColors,
       });
+      await tagcloudVisualization.render(dummyTableGroup, visParams);
 
       SVGElementGetBBoxSpyInstance.mockRestore();
       SVGElementGetBBoxSpyInstance = setSVGElementGetBBox(256, 368);
@@ -150,15 +121,9 @@ describe('TagCloudVisualizationTest', () => {
       HTMLElementOffsetMockInstance.mockRestore();
       HTMLElementOffsetMockInstance = setHTMLElementOffset(256, 386);
 
-      vis.params.orientation = 'right angled';
-      vis.params.minFontSize = 70;
-      await tagcloudVisualization.render(dummyTableGroup, vis.params, {
-        resize: true,
-        params: true,
-        aggs: false,
-        data: false,
-        uiState: false,
-      });
+      visParams.orientation = 'right angled';
+      visParams.minFontSize = 70;
+      await tagcloudVisualization.render(dummyTableGroup, visParams);
 
       const svgNode = domNode.querySelector('svg');
       expect(svgNode.outerHTML).toMatchSnapshot();
diff --git a/src/plugins/vis_type_tagcloud/public/index.scss b/src/plugins/vis_type_tagcloud/public/index.scss
deleted file mode 100644
index e6893b9a2474c3..00000000000000
--- a/src/plugins/vis_type_tagcloud/public/index.scss
+++ /dev/null
@@ -1,8 +0,0 @@
-// Prefix all styles with "tgc" to avoid conflicts.
-// Examples
-//   tgcChart
-//   tgcChart__legend
-//   tgcChart__legend--small
-//   tgcChart__legend-isLoading
-
-@import './tag_cloud';
diff --git a/src/plugins/vis_type_tagcloud/public/plugin.ts b/src/plugins/vis_type_tagcloud/public/plugin.ts
index 6978186058b1d6..f1ea965e2c806c 100644
--- a/src/plugins/vis_type_tagcloud/public/plugin.ts
+++ b/src/plugins/vis_type_tagcloud/public/plugin.ts
@@ -23,12 +23,11 @@ import { VisualizationsSetup } from '../../visualizations/public';
 import { ChartsPluginSetup } from '../../charts/public';
 
 import { createTagCloudFn } from './tag_cloud_fn';
-import { createTagCloudVisTypeDefinition } from './tag_cloud_type';
+import { tagCloudVisTypeDefinition } from './tag_cloud_type';
 import { DataPublicPluginStart } from '../../data/public';
 import { setFormatService } from './services';
 import { ConfigSchema } from '../config';
-
-import './index.scss';
+import { getTagCloudVisRenderer } from './tag_cloud_vis_renderer';
 
 /** @internal */
 export interface TagCloudPluginSetupDependencies {
@@ -63,9 +62,8 @@ export class TagCloudPlugin implements Plugin<void, void> {
       colors: charts.colors,
     };
     expressions.registerFunction(createTagCloudFn);
-    visualizations.createBaseVisualization(
-      createTagCloudVisTypeDefinition(visualizationDependencies)
-    );
+    expressions.registerRenderer(getTagCloudVisRenderer(visualizationDependencies));
+    visualizations.createBaseVisualization(tagCloudVisTypeDefinition);
   }
 
   public start(core: CoreStart, { data }: TagCloudVisPluginStartDependencies) {
diff --git a/src/plugins/vis_type_tagcloud/public/tag_cloud_fn.ts b/src/plugins/vis_type_tagcloud/public/tag_cloud_fn.ts
index 05cf05ab00b759..42e126908c00f7 100644
--- a/src/plugins/vis_type_tagcloud/public/tag_cloud_fn.ts
+++ b/src/plugins/vis_type_tagcloud/public/tag_cloud_fn.ts
@@ -26,22 +26,23 @@ const name = 'tagcloud';
 
 interface Arguments extends TagCloudVisParams {
   metric: any; // these aren't typed yet
-  bucket: any; // these aren't typed yet
+  bucket?: any; // these aren't typed yet
 }
 
-interface RenderValue {
+export interface TagCloudVisRenderValue {
   visType: typeof name;
   visData: KibanaDatatable;
-  visConfig: Arguments;
-  params: any;
+  visParams: Arguments;
 }
 
-export const createTagCloudFn = (): ExpressionFunctionDefinition<
+export type TagcloudExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof name,
   KibanaDatatable,
   Arguments,
-  Render<RenderValue>
-> => ({
+  Render<TagCloudVisRenderValue>
+>;
+
+export const createTagCloudFn = (): TagcloudExpressionFunctionDefinition => ({
   name,
   type: 'render',
   inputTypes: ['kibana_datatable'],
@@ -95,7 +96,7 @@ export const createTagCloudFn = (): ExpressionFunctionDefinition<
     },
   },
   fn(input, args) {
-    const visConfig = {
+    const visParams = {
       scale: args.scale,
       orientation: args.orientation,
       minFontSize: args.minFontSize,
@@ -105,19 +106,16 @@ export const createTagCloudFn = (): ExpressionFunctionDefinition<
     } as Arguments;
 
     if (args.bucket !== undefined) {
-      visConfig.bucket = args.bucket;
+      visParams.bucket = args.bucket;
     }
 
     return {
       type: 'render',
-      as: 'visualization',
+      as: 'tagloud_vis',
       value: {
         visData: input,
         visType: name,
-        visConfig,
-        params: {
-          listenOnChange: true,
-        },
+        visParams,
       },
     };
   },
diff --git a/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts b/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts
index 023489c6d2e876..d4c37649f949d5 100644
--- a/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts
+++ b/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts
@@ -20,15 +20,12 @@
 import { i18n } from '@kbn/i18n';
 
 import { Schemas } from '../../vis_default_editor/public';
+import { VIS_EVENT_TO_TRIGGER } from '../../visualizations/public';
 
 import { TagCloudOptions } from './components/tag_cloud_options';
-import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public';
+import { toExpressionAst } from './to_ast';
 
-// @ts-ignore
-import { createTagCloudVisualization } from './components/tag_cloud_visualization';
-import { TagCloudVisDependencies } from './plugin';
-
-export const createTagCloudVisTypeDefinition = (deps: TagCloudVisDependencies) => ({
+export const tagCloudVisTypeDefinition = {
   name: 'tagcloud',
   title: i18n.translate('visTypeTagCloud.vis.tagCloudTitle', { defaultMessage: 'Tag Cloud' }),
   icon: 'visTagCloud',
@@ -47,7 +44,7 @@ export const createTagCloudVisTypeDefinition = (deps: TagCloudVisDependencies) =
       showLabel: true,
     },
   },
-  visualization: createTagCloudVisualization(deps),
+  toExpressionAst,
   editorConfig: {
     collections: {
       scales: [
@@ -123,5 +120,4 @@ export const createTagCloudVisTypeDefinition = (deps: TagCloudVisDependencies) =
       },
     ]),
   },
-  useCustomNoDataScreen: true,
-});
+};
diff --git a/src/plugins/vis_type_tagcloud/public/tag_cloud_vis_renderer.tsx b/src/plugins/vis_type_tagcloud/public/tag_cloud_vis_renderer.tsx
new file mode 100644
index 00000000000000..d37aa5f6fe409a
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/tag_cloud_vis_renderer.tsx
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { lazy } from 'react';
+import { render, unmountComponentAtNode } from 'react-dom';
+
+import { VisualizationContainer } from '../../visualizations/public';
+import { ExpressionRenderDefinition } from '../../expressions/common/expression_renderers';
+import { TagCloudVisDependencies } from './plugin';
+import { TagCloudVisRenderValue } from './tag_cloud_fn';
+
+// @ts-ignore
+const TagCloudChart = lazy(() => import('./components/tag_cloud_chart'));
+
+export const getTagCloudVisRenderer: (
+  deps: TagCloudVisDependencies
+) => ExpressionRenderDefinition<TagCloudVisRenderValue> = ({ colors }) => ({
+  name: 'tagloud_vis',
+  displayName: 'Tag Cloud visualization',
+  reuseDomNode: true,
+  render: async (domNode, config, handlers) => {
+    handlers.onDestroy(() => {
+      unmountComponentAtNode(domNode);
+    });
+
+    render(
+      <VisualizationContainer>
+        <TagCloudChart
+          {...config}
+          colors={colors}
+          renderComplete={handlers.done}
+          fireEvent={handlers.event}
+        />
+      </VisualizationContainer>,
+      domNode
+    );
+  },
+});
diff --git a/src/plugins/vis_type_tagcloud/public/to_ast.test.ts b/src/plugins/vis_type_tagcloud/public/to_ast.test.ts
new file mode 100644
index 00000000000000..681f55bc2b3cdd
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/to_ast.test.ts
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Vis } from 'src/plugins/visualizations/public';
+import { toExpressionAst } from './to_ast';
+import { TagCloudVisParams } from './types';
+
+const mockSchemas = {
+  metric: [{ accessor: 1, format: { id: 'number' }, params: {}, label: 'Count', aggType: 'count' }],
+  segment: [
+    {
+      accessor: 0,
+      format: {
+        id: 'terms',
+        params: {
+          id: 'string',
+          otherBucketLabel: 'Other',
+          missingBucketLabel: 'Missing',
+        },
+      },
+      params: {},
+      label: 'products.product_name.keyword: Descending',
+      aggType: 'terms',
+    },
+  ],
+};
+
+jest.mock('../../visualizations/public', () => ({
+  getVisSchemas: () => mockSchemas,
+}));
+
+describe('tagcloud vis toExpressionAst function', () => {
+  let vis: Vis<TagCloudVisParams>;
+
+  beforeEach(() => {
+    vis = {
+      isHierarchical: () => false,
+      type: {},
+      params: {
+        showLabel: false,
+      },
+      data: {
+        indexPattern: { id: '123' },
+        aggs: {
+          getResponseAggs: () => [],
+          aggs: [],
+        },
+      },
+    } as any;
+  });
+
+  it('should match snapshot without params', () => {
+    const actual = toExpressionAst(vis, {} as any);
+    expect(actual).toMatchSnapshot();
+  });
+
+  it('should match snapshot params fulfilled', () => {
+    vis.params = {
+      scale: 'linear',
+      orientation: 'single',
+      minFontSize: 5,
+      maxFontSize: 15,
+      showLabel: true,
+    };
+    const actual = toExpressionAst(vis, {} as any);
+    expect(actual).toMatchSnapshot();
+  });
+});
diff --git a/src/plugins/vis_type_tagcloud/public/to_ast.ts b/src/plugins/vis_type_tagcloud/public/to_ast.ts
new file mode 100644
index 00000000000000..a284bba3073487
--- /dev/null
+++ b/src/plugins/vis_type_tagcloud/public/to_ast.ts
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { EsaggsExpressionFunctionDefinition } from '../../data/common/search/expressions';
+import { buildExpression, buildExpressionFunction } from '../../expressions/public';
+import { getVisSchemas, SchemaConfig, Vis, BuildPipelineParams } from '../../visualizations/public';
+import { TagcloudExpressionFunctionDefinition } from './tag_cloud_fn';
+import { TagCloudVisParams } from './types';
+
+const prepareDimension = (params: SchemaConfig) => {
+  const visdimension = buildExpressionFunction('visdimension', { accessor: params.accessor });
+
+  if (params.format) {
+    visdimension.addArgument('format', params.format.id);
+    visdimension.addArgument('formatParams', JSON.stringify(params.format.params));
+  }
+
+  return buildExpression([visdimension]);
+};
+
+export const toExpressionAst = (vis: Vis<TagCloudVisParams>, params: BuildPipelineParams) => {
+  const esaggs = buildExpressionFunction<EsaggsExpressionFunctionDefinition>('esaggs', {
+    index: vis.data.indexPattern!.id!,
+    metricsAtAllLevels: vis.isHierarchical(),
+    partialRows: vis.type.requiresPartialRows || false,
+    aggConfigs: JSON.stringify(vis.data.aggs!.aggs),
+    includeFormatHints: false,
+  });
+
+  const schemas = getVisSchemas(vis, params);
+
+  const tagcloud = buildExpressionFunction<TagcloudExpressionFunctionDefinition>('tagcloud', {
+    ...vis.params,
+    metric: prepareDimension(schemas.metric[0]),
+  });
+
+  if (schemas.segment) {
+    tagcloud.addArgument('bucket', prepareDimension(schemas.segment[0]));
+  }
+
+  const ast = buildExpression([esaggs, tagcloud]);
+
+  return ast.toAst();
+};
diff --git a/src/plugins/visualizations/public/index.ts b/src/plugins/visualizations/public/index.ts
index 3e3926bc5c8d84..99c13b42b8b282 100644
--- a/src/plugins/visualizations/public/index.ts
+++ b/src/plugins/visualizations/public/index.ts
@@ -42,7 +42,7 @@ export type VisualizeEmbeddableFactoryContract = PublicContract<VisualizeEmbedda
 export type VisualizeEmbeddableContract = PublicContract<VisualizeEmbeddable>;
 export { VisualizeInput } from './embeddable';
 export type ExprVis = ExprVisClass;
-export { SchemaConfig } from './legacy/build_pipeline';
+export { SchemaConfig, BuildPipelineParams } from './legacy/build_pipeline';
 // @ts-ignore
 export { updateOldState } from './legacy/vis_update_state';
 export { PersistedState } from './persisted_state';
diff --git a/src/plugins/visualizations/public/legacy/__snapshots__/build_pipeline.test.ts.snap b/src/plugins/visualizations/public/legacy/__snapshots__/build_pipeline.test.ts.snap
index fae777b98ef634..654ac78cdaa022 100644
--- a/src/plugins/visualizations/public/legacy/__snapshots__/build_pipeline.test.ts.snap
+++ b/src/plugins/visualizations/public/legacy/__snapshots__/build_pipeline.test.ts.snap
@@ -22,12 +22,6 @@ exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunct
 
 exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles table function without splits or buckets 1`] = `"kibana_table visConfig='{\\"foo\\":\\"bar\\",\\"dimensions\\":{\\"metrics\\":[{\\"accessor\\":0,\\"label\\":\\"\\",\\"format\\":{},\\"params\\":{},\\"aggType\\":\\"\\"},{\\"accessor\\":1,\\"label\\":\\"\\",\\"format\\":{},\\"params\\":{},\\"aggType\\":\\"\\"}],\\"buckets\\":[]}}' "`;
 
-exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles tagcloud function with boolean param showLabel 1`] = `"tagcloud metric={visdimension 0} showLabel=false "`;
-
-exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles tagcloud function with buckets 1`] = `"tagcloud metric={visdimension 0} bucket={visdimension 1 } "`;
-
-exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles tagcloud function without buckets 1`] = `"tagcloud metric={visdimension 0} "`;
-
 exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles tile_map function 1`] = `"tilemap visConfig='{\\"metric\\":{},\\"dimensions\\":{\\"metric\\":{\\"accessor\\":0,\\"label\\":\\"\\",\\"format\\":{},\\"params\\":{},\\"aggType\\":\\"\\"},\\"geohash\\":1,\\"geocentroid\\":3}}' "`;
 
 exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles timelion function 1`] = `"timelion_vis expression='foo' interval='bar' "`;
diff --git a/src/plugins/visualizations/public/legacy/build_pipeline.test.ts b/src/plugins/visualizations/public/legacy/build_pipeline.test.ts
index 2d92b386253b03..8cac76726b13b0 100644
--- a/src/plugins/visualizations/public/legacy/build_pipeline.test.ts
+++ b/src/plugins/visualizations/public/legacy/build_pipeline.test.ts
@@ -201,28 +201,6 @@ describe('visualize loader pipeline helpers: build pipeline', () => {
       });
     });
 
-    describe('handles tagcloud function', () => {
-      it('without buckets', () => {
-        const actual = buildPipelineVisFunction.tagcloud({}, schemasDef, uiState);
-        expect(actual).toMatchSnapshot();
-      });
-
-      it('with buckets', () => {
-        const schemas = {
-          ...schemasDef,
-          segment: [{ accessor: 1 }],
-        };
-        const actual = buildPipelineVisFunction.tagcloud({}, schemas, uiState);
-        expect(actual).toMatchSnapshot();
-      });
-
-      it('with boolean param showLabel', () => {
-        const params = { showLabel: false };
-        const actual = buildPipelineVisFunction.tagcloud(params, schemasDef, uiState);
-        expect(actual).toMatchSnapshot();
-      });
-    });
-
     describe('handles region_map function', () => {
       it('without buckets', () => {
         const params = { metric: {} };
diff --git a/src/plugins/visualizations/public/legacy/build_pipeline.ts b/src/plugins/visualizations/public/legacy/build_pipeline.ts
index 438a6d2337724e..dcc384a191858f 100644
--- a/src/plugins/visualizations/public/legacy/build_pipeline.ts
+++ b/src/plugins/visualizations/public/legacy/build_pipeline.ts
@@ -70,6 +70,12 @@ interface BuildVisConfigFunction {
   [key: string]: buildVisConfigFunction;
 }
 
+export interface BuildPipelineParams {
+  timefilter: TimefilterContract;
+  timeRange?: any;
+  abortSignal?: AbortSignal;
+}
+
 const vislibCharts: string[] = [
   'area',
   'gauge',
@@ -80,14 +86,7 @@ const vislibCharts: string[] = [
   'line',
 ];
 
-export const getSchemas = (
-  vis: Vis,
-  opts: {
-    timeRange?: any;
-    timefilter: TimefilterContract;
-  }
-): Schemas => {
-  const { timeRange, timefilter } = opts;
+export const getSchemas = (vis: Vis, { timeRange, timefilter }: BuildPipelineParams): Schemas => {
   const createSchemaConfig = (accessor: number, agg: IAggConfig): SchemaConfig => {
     if (isDateHistogramBucketAggConfig(agg)) {
       agg.params.timeRange = timeRange;
@@ -276,19 +275,6 @@ export const buildPipelineVisFunction: BuildPipelineVisFunction = {
     };
     return `kibana_table ${prepareJson('visConfig', visConfig)}`;
   },
-  tagcloud: (params, schemas) => {
-    const { scale, orientation, minFontSize, maxFontSize, showLabel } = params;
-    const { metric, bucket } = buildVisConfig.tagcloud(schemas);
-    let expr = `tagcloud metric={visdimension ${metric.accessor}} `;
-    expr += prepareValue('scale', scale);
-    expr += prepareValue('orientation', orientation);
-    expr += prepareValue('minFontSize', minFontSize);
-    expr += prepareValue('maxFontSize', maxFontSize);
-    expr += prepareValue('showLabel', showLabel);
-    expr += prepareDimension('bucket', bucket);
-
-    return expr;
-  },
   region_map: (params, schemas) => {
     const visConfig = {
       ...params,
@@ -333,14 +319,6 @@ const buildVisConfig: BuildVisConfigFunction = {
     }
     return visConfig;
   },
-  tagcloud: (schemas) => {
-    const visConfig = {} as any;
-    visConfig.metric = schemas.metric[0];
-    if (schemas.segment) {
-      visConfig.bucket = schemas.segment[0];
-    }
-    return visConfig;
-  },
   region_map: (schemas) => {
     const visConfig = {} as any;
     visConfig.metric = schemas.metric[0];
@@ -370,14 +348,7 @@ const buildVisConfig: BuildVisConfigFunction = {
   },
 };
 
-export const buildVislibDimensions = async (
-  vis: any,
-  params: {
-    timefilter: TimefilterContract;
-    timeRange?: any;
-    abortSignal?: AbortSignal;
-  }
-) => {
+export const buildVislibDimensions = async (vis: any, params: BuildPipelineParams) => {
   const schemas = getSchemas(vis, {
     timeRange: params.timeRange,
     timefilter: params.timefilter,
@@ -416,14 +387,7 @@ export const buildVislibDimensions = async (
   return dimensions;
 };
 
-export const buildPipeline = async (
-  vis: Vis,
-  params: {
-    timefilter: TimefilterContract;
-    timeRange?: any;
-    abortSignal?: AbortSignal;
-  }
-) => {
+export const buildPipeline = async (vis: Vis, params: BuildPipelineParams) => {
   const { indexPattern, searchSource } = vis.data;
   const query = searchSource!.getField('query');
   const filters = searchSource!.getField('filter');
@@ -455,10 +419,8 @@ export const buildPipeline = async (
     ${prepareJson('aggConfigs', vis.data.aggs!.aggs)} | `;
     }
 
-    const schemas = getSchemas(vis, {
-      timeRange: params.timeRange,
-      timefilter: params.timefilter,
-    });
+    const schemas = getSchemas(vis, params);
+
     if (buildPipelineVisFunction[vis.type.name]) {
       pipeline += buildPipelineVisFunction[vis.type.name](
         { title, ...vis.params },
diff --git a/src/plugins/visualizations/public/vis.ts b/src/plugins/visualizations/public/vis.ts
index b4fc9df1c6eccf..c6773e5a1bee3a 100644
--- a/src/plugins/visualizations/public/vis.ts
+++ b/src/plugins/visualizations/public/vis.ts
@@ -83,12 +83,12 @@ const getSearchSource = async (inputSearchSource: ISearchSource, savedSearchId?:
 
 type PartialVisState = Assign<SerializedVis, { data: Partial<SerializedVisData> }>;
 
-export class Vis {
+export class Vis<TVisParams = VisParams> {
   public readonly type: VisType;
   public readonly id?: string;
   public title: string = '';
   public description: string = '';
-  public params: VisParams = {};
+  public params: TVisParams;
   // Session state is for storing information that is transitory, and will not be saved with the visualization.
   // For instance, map bounds, which depends on the view port, browser window size, etc.
   public sessionState: Record<string, any> = {};
diff --git a/test/interpreter_functional/snapshots/baseline/partial_test_1.json b/test/interpreter_functional/snapshots/baseline/partial_test_1.json
index 6b2f93b47c0b28..9c642e5e266d0d 100644
--- a/test/interpreter_functional/snapshots/baseline/partial_test_1.json
+++ b/test/interpreter_functional/snapshots/baseline/partial_test_1.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/baseline/tagcloud_all_data.json b/test/interpreter_functional/snapshots/baseline/tagcloud_all_data.json
index ae1e817424cb10..153eea71dd8d15 100644
--- a/test/interpreter_functional/snapshots/baseline/tagcloud_all_data.json
+++ b/test/interpreter_functional/snapshots/baseline/tagcloud_all_data.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/baseline/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/baseline/tagcloud_fontsize.json
index c0da4794728807..d5f01afa468ab6 100644
--- a/test/interpreter_functional/snapshots/baseline/tagcloud_fontsize.json
+++ b/test/interpreter_functional/snapshots/baseline/tagcloud_fontsize.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/baseline/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/baseline/tagcloud_metric_data.json
index c5fbcd63b06854..72b5e957c19a57 100644
--- a/test/interpreter_functional/snapshots/baseline/tagcloud_metric_data.json
+++ b/test/interpreter_functional/snapshots/baseline/tagcloud_metric_data.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/baseline/tagcloud_options.json b/test/interpreter_functional/snapshots/baseline/tagcloud_options.json
index b67b0744494034..7cbe7cc79882fb 100644
--- a/test/interpreter_functional/snapshots/baseline/tagcloud_options.json
+++ b/test/interpreter_functional/snapshots/baseline/tagcloud_options.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/session/partial_test_1.json b/test/interpreter_functional/snapshots/session/partial_test_1.json
index 6b2f93b47c0b28..9c642e5e266d0d 100644
--- a/test/interpreter_functional/snapshots/session/partial_test_1.json
+++ b/test/interpreter_functional/snapshots/session/partial_test_1.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json
index ae1e817424cb10..153eea71dd8d15 100644
--- a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json
+++ b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json
index c0da4794728807..d5f01afa468ab6 100644
--- a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json
+++ b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json
index c5fbcd63b06854..72b5e957c19a57 100644
--- a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json
+++ b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file
diff --git a/test/interpreter_functional/snapshots/session/tagcloud_options.json b/test/interpreter_functional/snapshots/session/tagcloud_options.json
index b67b0744494034..7cbe7cc79882fb 100644
--- a/test/interpreter_functional/snapshots/session/tagcloud_options.json
+++ b/test/interpreter_functional/snapshots/session/tagcloud_options.json
@@ -1 +1 @@
-{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"tagcloud"}}
\ No newline at end of file
+{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"aggConfigParams":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"indexPatternId":"logstash-*","type":"terms"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"aggConfigParams":{},"indexPatternId":"logstash-*","type":"count"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}}
\ No newline at end of file

From 2defe88a2c0a0b57ffa2054c75f31dfac1c53b4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= <patryk.kopycinski@elastic.co>
Date: Mon, 28 Sep 2020 15:28:11 +0200
Subject: [PATCH 21/22] [Security Solution] Refactor useSelector (#75297)

---
 .../components/use_all_cases_modal/index.tsx  |  6 +++---
 .../navigation/use_get_url_search.tsx         |  5 ++---
 .../public/common/containers/source/index.tsx | 10 ++++------
 .../containers/use_full_screen/index.tsx      | 11 ++++++----
 .../containers/use_global_time/index.tsx      |  5 +++--
 .../public/common/hooks/use_selector.tsx      | 20 +++++++++++++++++++
 .../containers/authentications/index.tsx      |  9 ++++-----
 .../public/hosts/containers/hosts/index.tsx   |  4 ++--
 .../components/network_dns_table/index.tsx    |  8 +++-----
 .../components/network_http_table/index.tsx   |  9 ++++-----
 .../network_top_countries_table/index.tsx     |  9 ++++-----
 .../network_top_n_flow_table/index.tsx        |  9 ++++-----
 .../network/components/tls_table/index.tsx    |  9 ++++-----
 .../network/components/users_table/index.tsx  |  5 +++--
 .../network/containers/network_dns/index.tsx  |  9 +++------
 .../network/containers/network_http/index.tsx |  9 ++++-----
 .../network_top_countries/index.tsx           |  9 ++++-----
 .../containers/network_top_n_flow/index.tsx   |  9 ++++-----
 .../public/network/containers/tls/index.tsx   |  9 ++++-----
 .../public/network/containers/users/index.tsx |  4 ++--
 .../public/network/pages/details/index.tsx    |  7 ++++---
 .../components/graph_overlay/index.tsx        |  5 +++--
 .../components/open_timeline/index.tsx        |  8 +++-----
 .../row_renderers_browser/index.tsx           |  5 +++--
 .../timeline/body/actions/index.test.tsx      | 16 +++++++--------
 .../body/events/event_column_view.tsx         |  8 +++-----
 .../timeline/body/events/stateful_event.tsx   |  6 +++---
 .../components/timeline/body/index.test.tsx   | 12 +++--------
 .../add_data_provider_popover.tsx             |  5 +++--
 .../data_providers/provider_item_badge.tsx    |  5 +++--
 .../use_insert_timeline.tsx                   |  6 ++++--
 .../timeline/properties/helpers.tsx           |  6 +++---
 .../properties/use_create_timeline.tsx        |  9 +++++----
 33 files changed, 135 insertions(+), 131 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/public/common/hooks/use_selector.tsx

diff --git a/x-pack/plugins/security_solution/public/cases/components/use_all_cases_modal/index.tsx b/x-pack/plugins/security_solution/public/cases/components/use_all_cases_modal/index.tsx
index f7fc7963b3844b..445ae675007cc1 100644
--- a/x-pack/plugins/security_solution/public/cases/components/use_all_cases_modal/index.tsx
+++ b/x-pack/plugins/security_solution/public/cases/components/use_all_cases_modal/index.tsx
@@ -5,13 +5,13 @@
  */
 
 import React, { useState, useCallback, useMemo } from 'react';
+import { useDispatch } from 'react-redux';
 
-import { useDispatch, useSelector } from 'react-redux';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { APP_ID } from '../../../../common/constants';
 import { SecurityPageName } from '../../../app/types';
 import { useKibana } from '../../../common/lib/kibana';
 import { getCaseDetailsUrl, getCreateCaseUrl } from '../../../common/components/link_to';
-import { State } from '../../../common/store';
 import { setInsertTimeline } from '../../../timelines/store/timeline/actions';
 import { timelineSelectors } from '../../../timelines/store/timeline';
 
@@ -34,7 +34,7 @@ export const useAllCasesModal = ({
 }: UseAllCasesModalProps): UseAllCasesModalReturnedValues => {
   const dispatch = useDispatch();
   const { navigateToApp } = useKibana().services.application;
-  const timeline = useSelector((state: State) =>
+  const timeline = useShallowEqualSelector((state) =>
     timelineSelectors.selectTimeline(state, timelineId)
   );
 
diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/use_get_url_search.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/use_get_url_search.tsx
index 099c9ee3fca0ae..84f691dcd72e1d 100644
--- a/x-pack/plugins/security_solution/public/common/components/navigation/use_get_url_search.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/navigation/use_get_url_search.tsx
@@ -4,17 +4,16 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { isEqual } from 'lodash/fp';
 import { useMemo } from 'react';
-import { useSelector } from 'react-redux';
 
+import { useDeepEqualSelector } from '../../hooks/use_selector';
 import { makeMapStateToProps } from '../url_state/helpers';
 import { getSearch } from './helpers';
 import { SearchNavTab } from './types';
 
 export const useGetUrlSearch = (tab: SearchNavTab) => {
   const mapState = makeMapStateToProps();
-  const { urlState } = useSelector(mapState, isEqual);
+  const { urlState } = useDeepEqualSelector(mapState);
   const urlSearch = useMemo(() => getSearch(tab, urlState), [tab, urlState]);
   return urlSearch;
 };
diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx
index 4b1db8a2871bd8..c36e2de61fcbf1 100644
--- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx
@@ -8,7 +8,7 @@ import { set } from '@elastic/safer-lodash-set/fp';
 import { keyBy, pick, isEmpty, isEqual, isUndefined } from 'lodash/fp';
 import memoizeOne from 'memoize-one';
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import { IIndexPattern } from 'src/plugins/data/public';
 
 import { useKibana } from '../../lib/kibana';
@@ -20,11 +20,10 @@ import {
   BrowserFields,
 } from '../../../../common/search_strategy/index_fields';
 import { AbortError } from '../../../../../../../src/plugins/data/common';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import * as i18n from './translations';
 import { SourcererScopeName } from '../../store/sourcerer/model';
 import { sourcererActions, sourcererSelectors } from '../../store/sourcerer';
-
-import { State } from '../../store';
 import { DocValueFields } from '../../../../common/search_strategy/common';
 
 export { BrowserField, BrowserFields, DocValueFields };
@@ -201,9 +200,8 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => {
     () => sourcererSelectors.getIndexNamesSelectedSelector(),
     []
   );
-  const indexNames = useSelector<State, string[]>(
-    (state) => indexNamesSelectedSelector(state, sourcererScopeName),
-    shallowEqual
+  const indexNames = useShallowEqualSelector<string[]>((state) =>
+    indexNamesSelectedSelector(state, sourcererScopeName)
   );
 
   const setLoading = useCallback(
diff --git a/x-pack/plugins/security_solution/public/common/containers/use_full_screen/index.tsx b/x-pack/plugins/security_solution/public/common/containers/use_full_screen/index.tsx
index 32591fb032436d..8357a9d22739e4 100644
--- a/x-pack/plugins/security_solution/public/common/containers/use_full_screen/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/use_full_screen/index.tsx
@@ -5,9 +5,10 @@
  */
 
 import { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
-import { SCROLLING_DISABLED_CLASS_NAME } from '../../../../common/constants';
+import { useDispatch } from 'react-redux';
 
+import { SCROLLING_DISABLED_CLASS_NAME } from '../../../../common/constants';
+import { useShallowEqualSelector } from '../../hooks/use_selector';
 import { inputsSelectors } from '../../store';
 import { inputsActions } from '../../store/actions';
 
@@ -29,8 +30,10 @@ export const resetScroll = () => {
 
 export const useFullScreen = () => {
   const dispatch = useDispatch();
-  const globalFullScreen = useSelector(inputsSelectors.globalFullScreenSelector) ?? false;
-  const timelineFullScreen = useSelector(inputsSelectors.timelineFullScreenSelector) ?? false;
+  const globalFullScreen =
+    useShallowEqualSelector(inputsSelectors.globalFullScreenSelector) ?? false;
+  const timelineFullScreen =
+    useShallowEqualSelector(inputsSelectors.timelineFullScreenSelector) ?? false;
 
   const setGlobalFullScreen = useCallback(
     (fullScreen: boolean) => {
diff --git a/x-pack/plugins/security_solution/public/common/containers/use_global_time/index.tsx b/x-pack/plugins/security_solution/public/common/containers/use_global_time/index.tsx
index 52825caf9ce74a..e6c47c697c0b2e 100644
--- a/x-pack/plugins/security_solution/public/common/containers/use_global_time/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/use_global_time/index.tsx
@@ -5,15 +5,16 @@
  */
 
 import { useCallback, useState, useEffect, useMemo } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 
+import { useShallowEqualSelector } from '../../hooks/use_selector';
 import { inputsSelectors } from '../../store';
 import { inputsActions } from '../../store/actions';
 import { SetQuery, DeleteQuery } from './types';
 
 export const useGlobalTime = (clearAllQuery: boolean = true) => {
   const dispatch = useDispatch();
-  const { from, to } = useSelector(inputsSelectors.globalTimeRangeSelector);
+  const { from, to } = useShallowEqualSelector(inputsSelectors.globalTimeRangeSelector);
   const [isInitializing, setIsInitializing] = useState(true);
 
   const setQuery = useCallback(
diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_selector.tsx b/x-pack/plugins/security_solution/public/common/hooks/use_selector.tsx
new file mode 100644
index 00000000000000..d5416acb69d141
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/common/hooks/use_selector.tsx
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { shallowEqual, useSelector } from 'react-redux';
+import deepEqual from 'fast-deep-equal';
+import { State } from '../store';
+
+export type TypedUseSelectorHook = <TSelected, TState = State>(
+  selector: (state: TState) => TSelected,
+  equalityFn?: (left: TSelected, right: TSelected) => boolean
+) => TSelected;
+
+export const useShallowEqualSelector: TypedUseSelectorHook = (selector) =>
+  useSelector(selector, shallowEqual);
+
+export const useDeepEqualSelector: TypedUseSelectorHook = (selector) =>
+  useSelector(selector, deepEqual);
diff --git a/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx
index b1563e85c93ddf..bc7137097c6468 100644
--- a/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx
+++ b/x-pack/plugins/security_solution/public/hosts/containers/authentications/index.tsx
@@ -6,7 +6,6 @@
 
 import { noop } from 'lodash/fp';
 import { useCallback, useEffect, useRef, useState } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import {
@@ -26,7 +25,8 @@ import {
 } from '../../../../common/search_strategy';
 import { ESTermQuery } from '../../../../common/typed_json';
 
-import { inputsModel, State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
+import { inputsModel } from '../../../common/store';
 import { createFilter } from '../../../common/containers/helpers';
 import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers';
 import { useKibana } from '../../../common/lib/kibana';
@@ -71,9 +71,8 @@ export const useAuthentications = ({
   skip,
 }: UseAuthentications): [boolean, AuthenticationArgs] => {
   const getAuthenticationsSelector = hostsSelectors.authenticationsSelector();
-  const { activePage, limit } = useSelector(
-    (state: State) => getAuthenticationsSelector(state, type),
-    shallowEqual
+  const { activePage, limit } = useShallowEqualSelector((state) =>
+    getAuthenticationsSelector(state, type)
   );
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
diff --git a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx
index 6ca0272e58d7d6..77f4567fc6a5f8 100644
--- a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx
+++ b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx
@@ -7,11 +7,11 @@
 import deepEqual from 'fast-deep-equal';
 import { noop } from 'lodash/fp';
 import { useCallback, useEffect, useRef, useState } from 'react';
-import { useSelector } from 'react-redux';
 
 import { inputsModel, State } from '../../../common/store';
 import { createFilter } from '../../../common/containers/helpers';
 import { useKibana } from '../../../common/lib/kibana';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { hostsModel, hostsSelectors } from '../../store';
 import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers';
 import {
@@ -69,7 +69,7 @@ export const useAllHost = ({
   type,
 }: UseAllHost): [boolean, HostsArgs] => {
   const getHostsSelector = hostsSelectors.hostsSelector();
-  const { activePage, direction, limit, sortField } = useSelector((state: State) =>
+  const { activePage, direction, limit, sortField } = useShallowEqualSelector((state: State) =>
     getHostsSelector(state, type)
   );
   const { data, notifications } = useKibana().services;
diff --git a/x-pack/plugins/security_solution/public/network/components/network_dns_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/network_dns_table/index.tsx
index fa7690e9eeaff1..0d5b379a62d386 100644
--- a/x-pack/plugins/security_solution/public/network/components/network_dns_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/network_dns_table/index.tsx
@@ -5,7 +5,7 @@
  */
 
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { networkActions, networkModel, networkSelectors } from '../../store';
@@ -16,6 +16,7 @@ import {
   NetworkDnsFields,
 } from '../../../../common/search_strategy';
 import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/components/paginated_table';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 
 import { getNetworkDnsColumns } from './columns';
 import { IsPtrIncluded } from './is_ptr_included';
@@ -59,10 +60,7 @@ const NetworkDnsTableComponent: React.FC<NetworkDnsTableProps> = ({
 }) => {
   const dispatch = useDispatch();
   const getNetworkDnsSelector = networkSelectors.dnsSelector();
-  const { activePage, isPtrIncluded, limit, sort } = useSelector(
-    getNetworkDnsSelector,
-    shallowEqual
-  );
+  const { activePage, isPtrIncluded, limit, sort } = useShallowEqualSelector(getNetworkDnsSelector);
   const updateLimitPagination = useCallback(
     (newLimit) =>
       dispatch(
diff --git a/x-pack/plugins/security_solution/public/network/components/network_http_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/network_http_table/index.tsx
index ccb238bcee274d..6982388cafd9c2 100644
--- a/x-pack/plugins/security_solution/public/network/components/network_http_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/network_http_table/index.tsx
@@ -5,11 +5,11 @@
  */
 
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 
 import { networkActions, networkModel, networkSelectors } from '../../store';
 import { NetworkHttpEdges, NetworkHttpFields } from '../../../../common/search_strategy';
-import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/components/paginated_table';
 
 import { getNetworkHttpColumns } from './columns';
@@ -51,9 +51,8 @@ const NetworkHttpTableComponent: React.FC<NetworkHttpTableProps> = ({
 }) => {
   const dispatch = useDispatch();
   const getNetworkHttpSelector = networkSelectors.httpSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getNetworkHttpSelector(state, type),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getNetworkHttpSelector(state, type)
   );
   const tableType =
     type === networkModel.NetworkType.page
diff --git a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx
index 2495f9e7c11c81..9b265aa002cccb 100644
--- a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx
@@ -6,7 +6,7 @@
 
 import { last } from 'lodash/fp';
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 import { IIndexPattern } from 'src/plugins/data/public';
 
@@ -18,7 +18,7 @@ import {
   NetworkTopTablesFields,
   SortField,
 } from '../../../../common/search_strategy';
-import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 
 import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/components/paginated_table';
 
@@ -67,9 +67,8 @@ const NetworkTopCountriesTableComponent: React.FC<NetworkTopCountriesTableProps>
 }) => {
   const dispatch = useDispatch();
   const getTopCountriesSelector = networkSelectors.topCountriesSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTopCountriesSelector(state, type, flowTargeted),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTopCountriesSelector(state, type, flowTargeted)
   );
 
   const headerTitle: string = useMemo(
diff --git a/x-pack/plugins/security_solution/public/network/components/network_top_n_flow_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/network_top_n_flow_table/index.tsx
index 9824ac602bb43e..b1789569bed757 100644
--- a/x-pack/plugins/security_solution/public/network/components/network_top_n_flow_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/network_top_n_flow_table/index.tsx
@@ -5,7 +5,7 @@
  */
 import { last } from 'lodash/fp';
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import {
@@ -15,7 +15,7 @@ import {
   NetworkTopNFlowEdges,
   NetworkTopTablesFields,
 } from '../../../../common/search_strategy';
-import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/components/paginated_table';
 import { networkActions, networkModel, networkSelectors } from '../../store';
 import { getNFlowColumnsCurated } from './columns';
@@ -61,9 +61,8 @@ const NetworkTopNFlowTableComponent: React.FC<NetworkTopNFlowTableProps> = ({
 }) => {
   const dispatch = useDispatch();
   const getTopNFlowSelector = networkSelectors.topNFlowSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTopNFlowSelector(state, type, flowTargeted),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTopNFlowSelector(state, type, flowTargeted)
   );
 
   const columns = useMemo(
diff --git a/x-pack/plugins/security_solution/public/network/components/tls_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/tls_table/index.tsx
index 3d19eedc06a8e6..79590bdfa08702 100644
--- a/x-pack/plugins/security_solution/public/network/components/tls_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/tls_table/index.tsx
@@ -5,7 +5,7 @@
  */
 
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { networkActions, networkModel, networkSelectors } from '../../store';
@@ -15,7 +15,7 @@ import {
   NetworkTlsFields,
   SortField,
 } from '../../../../common/search_strategy';
-import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import {
   Criteria,
   ItemsPerRow,
@@ -63,9 +63,8 @@ const TlsTableComponent: React.FC<TlsTableProps> = ({
 }) => {
   const dispatch = useDispatch();
   const getTlsSelector = networkSelectors.tlsSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTlsSelector(state, type),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTlsSelector(state, type)
   );
   const tableType: networkModel.TopTlsTableType =
     type === networkModel.NetworkType.page
diff --git a/x-pack/plugins/security_solution/public/network/components/users_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/users_table/index.tsx
index 0355d0a30cfa49..7829449530829e 100644
--- a/x-pack/plugins/security_solution/public/network/components/users_table/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/users_table/index.tsx
@@ -5,9 +5,10 @@
  */
 
 import React, { useCallback, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { assertUnreachable } from '../../../../common/utility_types';
 import { networkActions, networkModel, networkSelectors } from '../../store';
 import {
@@ -68,7 +69,7 @@ const UsersTableComponent: React.FC<UsersTableProps> = ({
 }) => {
   const dispatch = useDispatch();
   const getUsersSelector = networkSelectors.usersSelector();
-  const { activePage, sort, limit } = useSelector(getUsersSelector, shallowEqual);
+  const { activePage, sort, limit } = useShallowEqualSelector(getUsersSelector);
   const updateLimitPagination = useCallback(
     (newLimit) =>
       dispatch(
diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx
index 1f199ba4f9acdc..c49aa6a415904f 100644
--- a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx
@@ -6,11 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { ESTermQuery } from '../../../../common/typed_json';
-import { inputsModel, State } from '../../../common/store';
+import { inputsModel } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { useKibana } from '../../../common/lib/kibana';
 import { createFilter } from '../../../common/containers/helpers';
 import { NetworkDnsEdges, PageInfoPaginated } from '../../../../common/search_strategy';
@@ -68,10 +68,7 @@ export const useNetworkDns = ({
   type,
 }: UseNetworkDns): [boolean, NetworkDnsArgs] => {
   const getNetworkDnsSelector = networkSelectors.dnsSelector();
-  const { activePage, sort, isPtrIncluded, limit } = useSelector(
-    (state: State) => getNetworkDnsSelector(state),
-    shallowEqual
-  );
+  const { activePage, sort, isPtrIncluded, limit } = useShallowEqualSelector(getNetworkDnsSelector);
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
   const abortCtrl = useRef(new AbortController());
diff --git a/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx
index 98202f6b42be6b..ec4ac39599351a 100644
--- a/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx
@@ -6,11 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { ESTermQuery } from '../../../../common/typed_json';
-import { inputsModel, State } from '../../../common/store';
+import { inputsModel } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { useKibana } from '../../../common/lib/kibana';
 import { createFilter } from '../../../common/containers/helpers';
 import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers';
@@ -68,9 +68,8 @@ export const useNetworkHttp = ({
   type,
 }: UseNetworkHttp): [boolean, NetworkHttpArgs] => {
   const getHttpSelector = networkSelectors.httpSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getHttpSelector(state, type),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getHttpSelector(state, type)
   );
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx
index 6b52966342e973..2d75de138a88c5 100644
--- a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx
@@ -6,11 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { ESTermQuery } from '../../../../common/typed_json';
-import { inputsModel, State } from '../../../common/store';
+import { inputsModel } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { useKibana } from '../../../common/lib/kibana';
 import { createFilter } from '../../../common/containers/helpers';
 import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers';
@@ -66,9 +66,8 @@ export const useNetworkTopCountries = ({
   type,
 }: UseNetworkTopCountries): [boolean, NetworkTopCountriesArgs] => {
   const getTopCountriesSelector = networkSelectors.topCountriesSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTopCountriesSelector(state, type, flowTarget),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTopCountriesSelector(state, type, flowTarget)
   );
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx
index d6dd14b3259f04..328bb5aabcbb85 100644
--- a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx
@@ -6,11 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { ESTermQuery } from '../../../../common/typed_json';
-import { inputsModel, State } from '../../../common/store';
+import { inputsModel } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { useKibana } from '../../../common/lib/kibana';
 import { createFilter } from '../../../common/containers/helpers';
 import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers';
@@ -66,9 +66,8 @@ export const useNetworkTopNFlow = ({
   type,
 }: UseNetworkTopNFlow): [boolean, NetworkTopNFlowArgs] => {
   const getTopNFlowSelector = networkSelectors.topNFlowSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTopNFlowSelector(state, type, flowTarget),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTopNFlowSelector(state, type, flowTarget)
   );
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
diff --git a/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx b/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx
index f40675a1255ff8..ddea2914a1bbbb 100644
--- a/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx
@@ -6,11 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
 import { ESTermQuery } from '../../../../common/typed_json';
-import { inputsModel, State } from '../../../common/store';
+import { inputsModel } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { useKibana } from '../../../common/lib/kibana';
 import { createFilter } from '../../../common/containers/helpers';
 import { PageInfoPaginated, FlowTargetSourceDest } from '../../../graphql/types';
@@ -67,9 +67,8 @@ export const useNetworkTls = ({
   type,
 }: UseNetworkTls): [boolean, NetworkTlsArgs] => {
   const getTlsSelector = networkSelectors.tlsSelector();
-  const { activePage, limit, sort } = useSelector(
-    (state: State) => getTlsSelector(state, type, flowTarget),
-    shallowEqual
+  const { activePage, limit, sort } = useShallowEqualSelector((state) =>
+    getTlsSelector(state, type, flowTarget)
   );
   const { data, notifications } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
diff --git a/x-pack/plugins/security_solution/public/network/containers/users/index.tsx b/x-pack/plugins/security_solution/public/network/containers/users/index.tsx
index a289f8d16e9b29..5bca8d773c2f62 100644
--- a/x-pack/plugins/security_solution/public/network/containers/users/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/containers/users/index.tsx
@@ -6,9 +6,9 @@
 
 import { noop } from 'lodash/fp';
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { shallowEqual, useSelector } from 'react-redux';
 import deepEqual from 'fast-deep-equal';
 
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { ESTermQuery } from '../../../../common/typed_json';
 import { DEFAULT_INDEX_KEY } from '../../../../common/constants';
 import { inputsModel } from '../../../common/store';
@@ -66,7 +66,7 @@ export const useNetworkUsers = ({
   startDate,
 }: UseNetworkUsers): [boolean, NetworkUsersArgs] => {
   const getNetworkUsersSelector = networkSelectors.usersSelector();
-  const { activePage, sort, limit } = useSelector(getNetworkUsersSelector, shallowEqual);
+  const { activePage, sort, limit } = useShallowEqualSelector(getNetworkUsersSelector);
   const { data, notifications, uiSettings } = useKibana().services;
   const refetch = useRef<inputsModel.Refetch>(noop);
   const abortCtrl = useRef(new AbortController());
diff --git a/x-pack/plugins/security_solution/public/network/pages/details/index.tsx b/x-pack/plugins/security_solution/public/network/pages/details/index.tsx
index a227dec4109150..bd563c2bd76177 100644
--- a/x-pack/plugins/security_solution/public/network/pages/details/index.tsx
+++ b/x-pack/plugins/security_solution/public/network/pages/details/index.tsx
@@ -6,9 +6,10 @@
 
 import { EuiHorizontalRule, EuiSpacer, EuiFlexItem } from '@elastic/eui';
 import React, { useCallback, useEffect, useMemo } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import { useParams } from 'react-router-dom';
 
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { FlowTarget, LastEventIndexKey } from '../../../../common/search_strategy';
 import { useGlobalTime } from '../../../common/containers/use_global_time';
 import { FiltersGlobal } from '../../../common/components/filters_global';
@@ -58,8 +59,8 @@ const NetworkDetailsComponent: React.FC = () => {
   const getGlobalQuerySelector = inputsSelectors.globalQuerySelector();
   const getGlobalFiltersQuerySelector = inputsSelectors.globalFiltersQuerySelector();
 
-  const query = useSelector(getGlobalQuerySelector, shallowEqual);
-  const filters = useSelector(getGlobalFiltersQuerySelector, shallowEqual);
+  const query = useShallowEqualSelector(getGlobalQuerySelector);
+  const filters = useShallowEqualSelector(getGlobalFiltersQuerySelector);
 
   const type = networkModel.NetworkType.details;
   const narrowDateRange = useCallback(
diff --git a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx
index dc9557da70f915..7b229b3fbb17e0 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx
@@ -14,7 +14,7 @@ import {
 } from '@elastic/eui';
 import { noop } from 'lodash/fp';
 import React, { useCallback, useMemo } from 'react';
-import { connect, ConnectedProps, useDispatch, useSelector } from 'react-redux';
+import { connect, ConnectedProps, useDispatch } from 'react-redux';
 import styled from 'styled-components';
 
 import { FULL_SCREEN } from '../timeline/body/column_headers/translations';
@@ -22,6 +22,7 @@ import { EXIT_FULL_SCREEN } from '../../../common/components/exit_full_screen/tr
 import { DEFAULT_INDEX_KEY, FULL_SCREEN_TOGGLED_CLASS_NAME } from '../../../../common/constants';
 import { useFullScreen } from '../../../common/containers/use_full_screen';
 import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { TimelineId, TimelineType } from '../../../../common/types/timeline';
 import { timelineSelectors } from '../../store/timeline';
 import { timelineDefaults } from '../../store/timeline/defaults';
@@ -109,7 +110,7 @@ const GraphOverlayComponent = ({
     dispatch(updateTimelineGraphEventId({ id: timelineId, graphEventId: '' }));
   }, [dispatch, timelineId]);
 
-  const currentTimeline = useSelector((state: State) =>
+  const currentTimeline = useShallowEqualSelector((state) =>
     timelineSelectors.selectTimeline(state, timelineId)
   );
 
diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/index.tsx
index dc824a8eb6272d..9498033ce2db3a 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/index.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/index.tsx
@@ -6,11 +6,12 @@
 
 import ApolloClient from 'apollo-client';
 import React, { useEffect, useState, useCallback, useMemo } from 'react';
-import { connect, ConnectedProps, shallowEqual, useSelector } from 'react-redux';
+import { connect, ConnectedProps } from 'react-redux';
 import { Dispatch } from 'redux';
 
 import { DeleteTimelineMutation, SortFieldTimeline, Direction } from '../../../graphql/types';
 import { sourcererSelectors, State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 import { TimelineId } from '../../../../common/types/timeline';
 import { ColumnHeaderOptions, TimelineModel } from '../../../timelines/store/timeline/model';
 import { timelineSelectors } from '../../../timelines/store/timeline';
@@ -114,10 +115,7 @@ export const StatefulOpenTimelineComponent = React.memo<OpenTimelineOwnProps>(
       () => sourcererSelectors.getAllExistingIndexNamesSelector(),
       []
     );
-    const existingIndexNames = useSelector<State, string[]>(
-      existingIndexNamesSelector,
-      shallowEqual
-    );
+    const existingIndexNames = useShallowEqualSelector<string[]>(existingIndexNamesSelector);
 
     const {
       customTemplateTimelineCount,
diff --git a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/index.tsx
index d01e8634a489fd..3c3ec1689b2447 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/index.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/index.tsx
@@ -20,10 +20,11 @@ import {
   EuiInMemoryTable,
 } from '@elastic/eui';
 import React, { useState, useCallback, useRef } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import styled from 'styled-components';
 
 import { State } from '../../../common/store';
+import { useShallowEqualSelector } from '../../../common/hooks/use_selector';
 
 import { renderers } from './catalog';
 import { setExcludedRowRendererIds as dispatchSetExcludedRowRendererIds } from '../../../timelines/store/timeline/actions';
@@ -81,7 +82,7 @@ const StatefulRowRenderersBrowserComponent: React.FC<StatefulRowRenderersBrowser
 }) => {
   const tableRef = useRef<EuiInMemoryTable<{}>>();
   const dispatch = useDispatch();
-  const excludedRowRendererIds = useSelector(
+  const excludedRowRendererIds = useShallowEqualSelector(
     (state: State) => state.timeline.timelineById[timelineId]?.excludedRowRendererIds || []
   );
   const [show, setShow] = useState(false);
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx
index fb1709df013205..0afca363096595 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx
@@ -5,22 +5,20 @@
  */
 import { mount } from 'enzyme';
 import React from 'react';
-import { useSelector } from 'react-redux';
 
 import { TestProviders, mockTimelineModel } from '../../../../../common/mock';
 import { DEFAULT_ACTIONS_COLUMN_WIDTH } from '../constants';
 import { Actions } from '.';
+import { useShallowEqualSelector } from '../../../../../common/hooks/use_selector';
 
-jest.mock('react-redux', () => {
-  const origin = jest.requireActual('react-redux');
-  return {
-    ...origin,
-    useSelector: jest.fn(),
-  };
-});
+jest.mock('../../../../../common/hooks/use_selector', () => ({
+  useShallowEqualSelector: jest.fn(),
+}));
 
 describe('Actions', () => {
-  (useSelector as jest.Mock).mockReturnValue(mockTimelineModel);
+  beforeEach(() => {
+    (useShallowEqualSelector as jest.Mock).mockReturnValue(mockTimelineModel);
+  });
 
   test('it renders a checkbox for selecting the event when `showCheckboxes` is `true`', () => {
     const wrapper = mount(
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx
index 2c84a786d4432a..f602ea1452e144 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx
@@ -6,8 +6,8 @@
 
 import React, { useCallback, useMemo } from 'react';
 import uuid from 'uuid';
-import { useSelector, shallowEqual } from 'react-redux';
 
+import { useShallowEqualSelector } from '../../../../../common/hooks/use_selector';
 import { Ecs } from '../../../../../../common/ecs';
 import { TimelineNonEcsData } from '../../../../../../common/search_strategy/timeline';
 import { Note } from '../../../../../common/lib/note';
@@ -28,7 +28,6 @@ import { AlertContextMenu } from '../../../../../detections/components/alerts_ta
 import { InvestigateInTimelineAction } from '../../../../../detections/components/alerts_table/timeline_actions/investigate_in_timeline_action';
 import { AddEventNoteAction } from '../actions/add_note_icon_item';
 import { PinEventAction } from '../actions/pin_event_action';
-import { StoreState } from '../../../../../common/store/types';
 import { inputsModel } from '../../../../../common/store';
 import { TimelineId } from '../../../../../../common/types/timeline';
 
@@ -96,9 +95,8 @@ export const EventColumnView = React.memo<Props>(
     toggleShowNotes,
     updateNote,
   }) => {
-    const { timelineType, status } = useSelector<StoreState, TimelineModel>(
-      (state) => state.timeline.timelineById[timelineId],
-      shallowEqual
+    const { timelineType, status } = useShallowEqualSelector<TimelineModel>(
+      (state) => state.timeline.timelineById[timelineId]
     );
 
     const handlePinClicked = useCallback(
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/stateful_event.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/stateful_event.tsx
index d8cc8ddd2f15a4..d71af86c802476 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/stateful_event.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/stateful_event.tsx
@@ -5,11 +5,11 @@
  */
 
 import React, { useRef, useState, useCallback } from 'react';
-import { useSelector } from 'react-redux';
 import uuid from 'uuid';
 import VisibilitySensor from 'react-visibility-sensor';
 
 import { BrowserFields, DocValueFields } from '../../../../../common/containers/source';
+import { useShallowEqualSelector } from '../../../../../common/hooks/use_selector';
 import { useTimelineEventsDetails } from '../../../../containers/details';
 import {
   TimelineEventsDetailsItem,
@@ -37,7 +37,7 @@ import { getEventType } from '../helpers';
 import { NoteCards } from '../../../notes/note_cards';
 import { useEventDetailsWidthContext } from '../../../../../common/components/events_viewer/event_details_width_context';
 import { EventColumnView } from './event_column_view';
-import { inputsModel, StoreState } from '../../../../../common/store';
+import { inputsModel } from '../../../../../common/store';
 
 interface Props {
   actionsColumnWidth: number;
@@ -136,7 +136,7 @@ const StatefulEventComponent: React.FC<Props> = ({
 }) => {
   const [expanded, setExpanded] = useState<{ [eventId: string]: boolean }>({});
   const [showNotes, setShowNotes] = useState<{ [eventId: string]: boolean }>({});
-  const { status: timelineStatus } = useSelector<StoreState, TimelineModel>(
+  const { status: timelineStatus } = useShallowEqualSelector<TimelineModel>(
     (state) => state.timeline.timelineById[timelineId]
   );
   const divElement = useRef<HTMLDivElement | null>(null);
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/index.test.tsx
index 23a449cca972d9..8b859c1746a468 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/index.test.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/index.test.tsx
@@ -5,7 +5,6 @@
  */
 import { ReactWrapper } from 'enzyme';
 import React from 'react';
-import { useSelector } from 'react-redux';
 
 import '../../../../common/mock/match_media';
 import { mockBrowserFields } from '../../../../common/containers/source/mock';
@@ -28,13 +27,9 @@ const mockSort: Sort = {
   sortDirection: Direction.desc,
 };
 
-jest.mock('react-redux', () => {
-  const origin = jest.requireActual('react-redux');
-  return {
-    ...origin,
-    useSelector: jest.fn(),
-  };
-});
+jest.mock('../../../../common/hooks/use_selector', () => ({
+  useShallowEqualSelector: jest.fn().mockReturnValue(mockTimelineModel),
+}));
 
 jest.mock('../../../../common/components/link_to');
 
@@ -87,7 +82,6 @@ describe('Body', () => {
     toggleColumn: jest.fn(),
     updateNote: jest.fn(),
   };
-  (useSelector as jest.Mock).mockReturnValue(mockTimelineModel);
 
   describe('rendering', () => {
     test('it renders the column headers', () => {
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/add_data_provider_popover.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/add_data_provider_popover.tsx
index 1bfeb482873c73..3523e8c0d7aaf8 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/add_data_provider_popover.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/add_data_provider_popover.tsx
@@ -15,10 +15,11 @@ import {
   EuiContextMenuPanelItemDescriptor,
 } from '@elastic/eui';
 import uuid from 'uuid';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 
 import { BrowserFields } from '../../../../common/containers/source';
 import { TimelineType } from '../../../../../common/types/timeline';
+import { useShallowEqualSelector } from '../../../../common/hooks/use_selector';
 import { StatefulEditDataProvider } from '../../edit_data_provider';
 import { addContentToTimeline } from './helpers';
 import { DataProviderType } from './data_provider';
@@ -36,7 +37,7 @@ const AddDataProviderPopoverComponent: React.FC<AddDataProviderPopoverProps> = (
 }) => {
   const dispatch = useDispatch();
   const [isAddFilterPopoverOpen, setIsAddFilterPopoverOpen] = useState(false);
-  const timelineById = useSelector(timelineSelectors.timelineByIdSelector);
+  const timelineById = useShallowEqualSelector(timelineSelectors.timelineByIdSelector);
   const { dataProviders, timelineType } = timelineById[timelineId] ?? {};
 
   const handleOpenPopover = useCallback(() => setIsAddFilterPopoverOpen(true), [
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/provider_item_badge.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/provider_item_badge.tsx
index ece23d7a108861..0093c43a17137e 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/provider_item_badge.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/provider_item_badge.tsx
@@ -6,10 +6,11 @@
 
 import { noop } from 'lodash/fp';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 
 import { TimelineType } from '../../../../../common/types/timeline';
 import { BrowserFields } from '../../../../common/containers/source';
+import { useShallowEqualSelector } from '../../../../common/hooks/use_selector';
 import { timelineSelectors } from '../../../store/timeline';
 
 import { OnDataProviderEdited } from '../events';
@@ -59,7 +60,7 @@ export const ProviderItemBadge = React.memo<ProviderItemBadgeProps>(
     val,
     type = DataProviderType.default,
   }) => {
-    const timelineById = useSelector(timelineSelectors.timelineByIdSelector);
+    const timelineById = useShallowEqualSelector(timelineSelectors.timelineByIdSelector);
     const timelineType = timelineId ? timelineById[timelineId]?.timelineType : TimelineType.default;
     const { getManageTimelineById } = useManageTimeline();
     const isLoading = useMemo(() => getManageTimelineById(timelineId ?? '').isLoading, [
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/use_insert_timeline.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/use_insert_timeline.tsx
index f1f419fd4b52a2..6a65819a764eba 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/use_insert_timeline.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/use_insert_timeline.tsx
@@ -5,7 +5,9 @@
  */
 
 import { useCallback, useState, useEffect } from 'react';
-import { useDispatch, useSelector, shallowEqual } from 'react-redux';
+import { useDispatch } from 'react-redux';
+
+import { useShallowEqualSelector } from '../../../../common/hooks/use_selector';
 import { SecurityPageName } from '../../../../../common/constants';
 import { getTimelineUrl, useFormatUrl } from '../../../../common/components/link_to';
 import { CursorPosition } from '../../../../common/components/markdown_editor';
@@ -20,7 +22,7 @@ export const useInsertTimeline = (value: string, onChange: (newValue: string) =>
     end: 0,
   });
 
-  const insertTimeline = useSelector(timelineSelectors.selectInsertTimeline, shallowEqual);
+  const insertTimeline = useShallowEqualSelector(timelineSelectors.selectInsertTimeline);
 
   const handleOnTimelineChange = useCallback(
     (title: string, id: string | null, graphEventId?: string) => {
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/helpers.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/helpers.tsx
index 0f6535015c799b..43ab8ab203e116 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/helpers.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/helpers.tsx
@@ -20,7 +20,7 @@ import {
 import React, { useCallback, useMemo } from 'react';
 import uuid from 'uuid';
 import styled from 'styled-components';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 
 import { APP_ID } from '../../../../../common/constants';
 import {
@@ -33,9 +33,9 @@ import {
 import { SecurityPageName } from '../../../../app/types';
 import { timelineSelectors } from '../../../../timelines/store/timeline';
 import { getCreateCaseUrl } from '../../../../common/components/link_to';
-import { State } from '../../../../common/store';
 import { useKibana } from '../../../../common/lib/kibana';
 import { Note } from '../../../../common/lib/note';
+import { useShallowEqualSelector } from '../../../../common/hooks/use_selector';
 
 import { Notes } from '../../notes';
 import { AssociateNote, UpdateNote } from '../../notes/helpers';
@@ -159,7 +159,7 @@ interface NewCaseProps {
 export const NewCase = React.memo<NewCaseProps>(
   ({ compact, graphEventId, onClosePopover, timelineId, timelineStatus, timelineTitle }) => {
     const dispatch = useDispatch();
-    const { savedObjectId } = useSelector((state: State) =>
+    const { savedObjectId } = useShallowEqualSelector((state) =>
       timelineSelectors.selectTimeline(state, timelineId)
     );
     const { navigateToApp } = useKibana().services.application;
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/use_create_timeline.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/use_create_timeline.tsx
index 3919ee21b2a907..28dd865c763ae4 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/use_create_timeline.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/properties/use_create_timeline.tsx
@@ -4,8 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import React, { useCallback, useMemo } from 'react';
-import { shallowEqual, useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import { EuiButton, EuiButtonEmpty } from '@elastic/eui';
+
 import { defaultHeaders } from '../body/column_headers/default_headers';
 import { timelineActions } from '../../../store/timeline';
 import { useFullScreen } from '../../../../common/containers/use_full_screen';
@@ -14,9 +15,9 @@ import {
   TimelineType,
   TimelineTypeLiteral,
 } from '../../../../../common/types/timeline';
+import { useShallowEqualSelector } from '../../../../common/hooks/use_selector';
 import { inputsActions, inputsSelectors } from '../../../../common/store/inputs';
 import { sourcererActions, sourcererSelectors } from '../../../../common/store/sourcerer';
-import { State } from '../../../../common/store';
 import { SourcererScopeName } from '../../../../common/store/sourcerer/model';
 
 export const useCreateTimelineButton = ({
@@ -33,9 +34,9 @@ export const useCreateTimelineButton = ({
     () => sourcererSelectors.getAllExistingIndexNamesSelector(),
     []
   );
-  const existingIndexNames = useSelector<State, string[]>(existingIndexNamesSelector, shallowEqual);
+  const existingIndexNames = useShallowEqualSelector<string[]>(existingIndexNamesSelector);
   const { timelineFullScreen, setTimelineFullScreen } = useFullScreen();
-  const globalTimeRange = useSelector(inputsSelectors.globalTimeRangeSelector);
+  const globalTimeRange = useShallowEqualSelector(inputsSelectors.globalTimeRangeSelector);
   const createTimeline = useCallback(
     ({ id, show }) => {
       if (id === TimelineId.active && timelineFullScreen) {

From aa9774faa08a2d85a45f5de35fbcbcee738dac5d Mon Sep 17 00:00:00 2001
From: Pierre Gayvallet <pierre.gayvallet@elastic.co>
Date: Mon, 28 Sep 2020 15:30:59 +0200
Subject: [PATCH 22/22] Add the @kbn/apm-config-loader package (#77855)

* first shot of the apm configuration loader

* revert changes to kibana config

* remove test files for now

* remove `?.` usages

* use lazy config init to avoid crashing integration test runner

* loader improvements

* add config value override via cli args

* add tests for utils package

* add prod/dev config handling + loader tests

* add tests for config

* address josh's comments

* nit on doc
---
 package.json                                  |   1 +
 packages/kbn-apm-config-loader/README.md      |  13 ++
 .../__fixtures__/config.yml                   |  11 ++
 .../__fixtures__/config_flat.yml              |   6 +
 .../__fixtures__/en_var_ref_config.yml        |   5 +
 .../__fixtures__/one.yml                      |   9 +
 .../__fixtures__/two.yml                      |  10 ++
 packages/kbn-apm-config-loader/package.json   |  23 +++
 .../src/config.test.mocks.ts                  |  66 ++++++++
 .../kbn-apm-config-loader/src/config.test.ts  | 158 ++++++++++++++++++
 packages/kbn-apm-config-loader/src/config.ts  | 139 +++++++++++++++
 .../src/config_loader.test.mocks.ts           |  45 +++++
 .../src/config_loader.test.ts                 |  75 +++++++++
 .../src/config_loader.ts                      |  39 +++++
 packages/kbn-apm-config-loader/src/index.ts   |  22 +++
 packages/kbn-apm-config-loader/src/types.ts   |  24 +++
 .../__snapshots__/read_config.test.ts.snap    | 108 ++++++++++++
 .../src/utils/apply_config_overrides.test.ts  |  58 +++++++
 .../src/utils/apply_config_overrides.ts       |  38 +++++
 .../src/utils/ensure_deep_object.test.ts      | 156 +++++++++++++++++
 .../src/utils/ensure_deep_object.ts           |  61 +++++++
 .../src/utils/get_config_file_paths.test.ts   |  39 +++++
 .../src/utils/get_config_file_paths.ts        |  37 ++++
 .../kbn-apm-config-loader/src/utils/index.ts  |  22 +++
 .../src/utils/read_argv.test.ts               |  80 +++++++++
 .../src/utils/read_argv.ts                    |  36 ++++
 .../src/utils/read_config.test.ts             |  79 +++++++++
 .../src/utils/read_config.ts                  |  64 +++++++
 packages/kbn-apm-config-loader/tsconfig.json  |  12 ++
 packages/kbn-apm-config-loader/yarn.lock      |   1 +
 src/apm.js                                    |  82 +++------
 31 files changed, 1456 insertions(+), 63 deletions(-)
 create mode 100644 packages/kbn-apm-config-loader/README.md
 create mode 100644 packages/kbn-apm-config-loader/__fixtures__/config.yml
 create mode 100644 packages/kbn-apm-config-loader/__fixtures__/config_flat.yml
 create mode 100644 packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml
 create mode 100644 packages/kbn-apm-config-loader/__fixtures__/one.yml
 create mode 100644 packages/kbn-apm-config-loader/__fixtures__/two.yml
 create mode 100644 packages/kbn-apm-config-loader/package.json
 create mode 100644 packages/kbn-apm-config-loader/src/config.test.mocks.ts
 create mode 100644 packages/kbn-apm-config-loader/src/config.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/config.ts
 create mode 100644 packages/kbn-apm-config-loader/src/config_loader.test.mocks.ts
 create mode 100644 packages/kbn-apm-config-loader/src/config_loader.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/config_loader.ts
 create mode 100644 packages/kbn-apm-config-loader/src/index.ts
 create mode 100644 packages/kbn-apm-config-loader/src/types.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap
 create mode 100644 packages/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/apply_config_overrides.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/ensure_deep_object.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/index.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/read_argv.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/read_argv.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/read_config.test.ts
 create mode 100644 packages/kbn-apm-config-loader/src/utils/read_config.ts
 create mode 100644 packages/kbn-apm-config-loader/tsconfig.json
 create mode 120000 packages/kbn-apm-config-loader/yarn.lock

diff --git a/package.json b/package.json
index 10083ff07fccd1..505fac1df935ee 100644
--- a/package.json
+++ b/package.json
@@ -127,6 +127,7 @@
     "@hapi/good-squeeze": "5.2.1",
     "@hapi/wreck": "^15.0.2",
     "@kbn/analytics": "1.0.0",
+    "@kbn/apm-config-loader": "1.0.0",
     "@kbn/babel-preset": "1.0.0",
     "@kbn/config": "1.0.0",
     "@kbn/config-schema": "1.0.0",
diff --git a/packages/kbn-apm-config-loader/README.md b/packages/kbn-apm-config-loader/README.md
new file mode 100644
index 00000000000000..51623dc745f2c4
--- /dev/null
+++ b/packages/kbn-apm-config-loader/README.md
@@ -0,0 +1,13 @@
+# @kbn/apm-config-loader
+
+Configuration loader for the APM instrumentation script.
+
+This module is only meant to be used by the APM instrumentation script (`src/apm.js`)
+to load the required configuration options from the `kibana.yaml` configuration file with
+default values.
+
+### Why not just use @kbn-config?
+
+`@kbn/config` is the recommended way to load and read the kibana configuration file,
+however in the specific case of APM, we want to only need the minimal dependencies
+before loading `elastic-apm-node` to avoid losing instrumentation on the already loaded modules.
\ No newline at end of file
diff --git a/packages/kbn-apm-config-loader/__fixtures__/config.yml b/packages/kbn-apm-config-loader/__fixtures__/config.yml
new file mode 100644
index 00000000000000..b0706d8ff8ea09
--- /dev/null
+++ b/packages/kbn-apm-config-loader/__fixtures__/config.yml
@@ -0,0 +1,11 @@
+pid:
+  enabled: true
+  file: '/var/run/kibana.pid'
+  obj: { val: 3 }
+  arr: [1]
+  empty_obj: {}
+  empty_arr: []
+obj: { val: 3 }
+arr: [1, 2]
+empty_obj: {}
+empty_arr: []
diff --git a/packages/kbn-apm-config-loader/__fixtures__/config_flat.yml b/packages/kbn-apm-config-loader/__fixtures__/config_flat.yml
new file mode 100644
index 00000000000000..a687a9a9088bf2
--- /dev/null
+++ b/packages/kbn-apm-config-loader/__fixtures__/config_flat.yml
@@ -0,0 +1,6 @@
+pid.enabled: true
+pid.file: '/var/run/kibana.pid'
+pid.obj: { val: 3 }
+pid.arr: [1, 2]
+pid.empty_obj: {}
+pid.empty_arr: []
diff --git a/packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml b/packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml
new file mode 100644
index 00000000000000..761f6a43ba4520
--- /dev/null
+++ b/packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml
@@ -0,0 +1,5 @@
+foo: 1
+bar: "pre-${KBN_ENV_VAR1}-mid-${KBN_ENV_VAR2}-post"
+
+elasticsearch:
+  requestHeadersWhitelist: ["${KBN_ENV_VAR1}", "${KBN_ENV_VAR2}"]
diff --git a/packages/kbn-apm-config-loader/__fixtures__/one.yml b/packages/kbn-apm-config-loader/__fixtures__/one.yml
new file mode 100644
index 00000000000000..ccef51b5461949
--- /dev/null
+++ b/packages/kbn-apm-config-loader/__fixtures__/one.yml
@@ -0,0 +1,9 @@
+foo: 1
+bar: true
+xyz: ['1', '2']
+empty_arr: []
+abc:
+  def: test
+  qwe: 1
+  zyx: { val: 1 }
+pom.bom: 3
diff --git a/packages/kbn-apm-config-loader/__fixtures__/two.yml b/packages/kbn-apm-config-loader/__fixtures__/two.yml
new file mode 100644
index 00000000000000..a2ec41265d50f7
--- /dev/null
+++ b/packages/kbn-apm-config-loader/__fixtures__/two.yml
@@ -0,0 +1,10 @@
+foo: 2
+baz: bonkers
+xyz: ['3', '4']
+arr: [1]
+empty_arr: []
+abc:
+  ghi: test2
+  qwe: 2
+  zyx: {}
+pom.mob: 4
diff --git a/packages/kbn-apm-config-loader/package.json b/packages/kbn-apm-config-loader/package.json
new file mode 100644
index 00000000000000..1982ccdeda0ff9
--- /dev/null
+++ b/packages/kbn-apm-config-loader/package.json
@@ -0,0 +1,23 @@
+{
+  "name": "@kbn/apm-config-loader",
+  "main": "./target/index.js",
+  "types": "./target/index.d.ts",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "dependencies": {
+    "@elastic/safer-lodash-set": "0.0.0",
+    "@kbn/utils": "1.0.0",
+    "js-yaml": "3.13.1",
+    "lodash": "^4.17.20"
+  },
+  "devDependencies": {
+    "typescript": "4.0.2",
+    "tsd": "^0.7.4"
+  }
+}
diff --git a/packages/kbn-apm-config-loader/src/config.test.mocks.ts b/packages/kbn-apm-config-loader/src/config.test.mocks.ts
new file mode 100644
index 00000000000000..a0422665a55d2d
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config.test.mocks.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { join } from 'path';
+const childProcessModule = jest.requireActual('child_process');
+const fsModule = jest.requireActual('fs');
+
+export const mockedRootDir = '/root';
+
+export const packageMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true });
+
+export const devConfigMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, {
+  virtual: true,
+});
+
+export const gitRevExecMock = jest.fn();
+jest.doMock('child_process', () => ({
+  ...childProcessModule,
+  execSync: (command: string, options: any) => {
+    if (command.startsWith('git rev-parse')) {
+      return gitRevExecMock(command, options);
+    }
+    return childProcessModule.execSync(command, options);
+  },
+}));
+
+export const readUuidFileMock = jest.fn();
+jest.doMock('fs', () => ({
+  ...fsModule,
+  readFileSync: (path: string, options: any) => {
+    if (path.endsWith('uuid')) {
+      return readUuidFileMock(path, options);
+    }
+    return fsModule.readFileSync(path, options);
+  },
+}));
+
+export const resetAllMocks = () => {
+  packageMock.raw = {};
+  devConfigMock.raw = {};
+  gitRevExecMock.mockReset();
+  readUuidFileMock.mockReset();
+  jest.resetModules();
+};
diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts
new file mode 100644
index 00000000000000..fe6247673e3126
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config.test.ts
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  packageMock,
+  mockedRootDir,
+  gitRevExecMock,
+  devConfigMock,
+  readUuidFileMock,
+  resetAllMocks,
+} from './config.test.mocks';
+
+import { ApmConfiguration } from './config';
+
+describe('ApmConfiguration', () => {
+  beforeEach(() => {
+    packageMock.raw = {
+      version: '8.0.0',
+      build: {
+        sha: 'sha',
+      },
+    };
+  });
+
+  afterEach(() => {
+    resetAllMocks();
+  });
+
+  it('sets the correct service name', () => {
+    packageMock.raw = {
+      version: '9.2.1',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('myservice').serviceName).toBe('myservice-9_2_1');
+  });
+
+  it('sets the git revision from `git rev-parse` command in non distribution mode', () => {
+    gitRevExecMock.mockReturnValue('some-git-rev');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('some-git-rev');
+  });
+
+  it('sets the git revision from `pkg.build.sha` in distribution mode', () => {
+    gitRevExecMock.mockReturnValue('dev-sha');
+    packageMock.raw = {
+      version: '9.2.1',
+      build: {
+        sha: 'distribution-sha',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('distribution-sha');
+  });
+
+  it('reads the kibana uuid from the uuid file', () => {
+    readUuidFileMock.mockReturnValue('instance-uuid');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('instance-uuid');
+  });
+
+  it('uses the uuid from the kibana config if present', () => {
+    readUuidFileMock.mockReturnValue('uuid-from-file');
+    const kibanaConfig = {
+      server: {
+        uuid: 'uuid-from-config',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('uuid-from-config');
+  });
+
+  it('uses the correct default config depending on the `isDistributable` parameter', () => {
+    let config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        serverUrl: expect.any(String),
+        secretToken: expect.any(String),
+      })
+    );
+
+    config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl');
+  });
+
+  it('loads the configuration from the kibana config file', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://url',
+        secretToken: 'secret',
+      })
+    );
+  });
+
+  it('loads the configuration from the dev config is present', () => {
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+      })
+    );
+  });
+
+  it('respect the precedence of the dev config', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+        secretToken: 'secret',
+      })
+    );
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts
new file mode 100644
index 00000000000000..aab82c6c06a582
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config.ts
@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { join } from 'path';
+import { merge, get } from 'lodash';
+import { execSync } from 'child_process';
+// deep import to avoid loading the whole package
+import { getDataPath } from '@kbn/utils/target/path';
+import { readFileSync } from 'fs';
+import { ApmAgentConfig } from './types';
+
+const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => {
+  if (isDistributable) {
+    return {
+      active: false,
+      globalLabels: {},
+    };
+  }
+  return {
+    active: false,
+    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
+    // The secretToken below is intended to be hardcoded in this file even though
+    // it makes it public. This is not a security/privacy issue. Normally we'd
+    // instead disable the need for a secretToken in the APM Server config where
+    // the data is transmitted to, but due to how it's being hosted, it's easier,
+    // for now, to simply leave it in.
+    secretToken: 'R0Gjg46pE9K9wGestd',
+    globalLabels: {},
+    breakdownMetrics: true,
+    centralConfig: false,
+    logUncaughtExceptions: true,
+  };
+};
+
+export class ApmConfiguration {
+  private baseConfig?: any;
+  private kibanaVersion: string;
+  private pkgBuild: Record<string, any>;
+
+  constructor(
+    private readonly rootDir: string,
+    private readonly rawKibanaConfig: Record<string, any>,
+    private readonly isDistributable: boolean
+  ) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { version, build } = require(join(this.rootDir, 'package.json'));
+    this.kibanaVersion = version.replace(/\./g, '_');
+    this.pkgBuild = build;
+  }
+
+  public getConfig(serviceName: string): ApmAgentConfig {
+    return {
+      ...this.getBaseConfig(),
+      serviceName: `${serviceName}-${this.kibanaVersion}`,
+    };
+  }
+
+  private getBaseConfig() {
+    if (!this.baseConfig) {
+      const apmConfig = merge(
+        getDefaultConfig(this.isDistributable),
+        this.getConfigFromKibanaConfig(),
+        this.getDevConfig()
+      );
+
+      const rev = this.getGitRev();
+      if (rev !== null) {
+        apmConfig.globalLabels.git_rev = rev;
+      }
+
+      const uuid = this.getKibanaUuid();
+      if (uuid) {
+        apmConfig.globalLabels.kibana_uuid = uuid;
+      }
+      this.baseConfig = apmConfig;
+    }
+
+    return this.baseConfig;
+  }
+
+  private getConfigFromKibanaConfig(): ApmAgentConfig {
+    return get(this.rawKibanaConfig, 'elastic.apm', {});
+  }
+
+  private getKibanaUuid() {
+    // try to access the `server.uuid` value from the config file first.
+    // if not manually defined, we will then read the value from the `{DATA_FOLDER}/uuid` file.
+    // note that as the file is created by the platform AFTER apm init, the file
+    // will not be present at first startup, but there is nothing we can really do about that.
+    if (get(this.rawKibanaConfig, 'server.uuid')) {
+      return this.rawKibanaConfig.server.uuid;
+    }
+
+    const dataPath: string = get(this.rawKibanaConfig, 'path.data') || getDataPath();
+    try {
+      const filename = join(dataPath, 'uuid');
+      return readFileSync(filename, 'utf-8');
+    } catch (e) {} // eslint-disable-line no-empty
+  }
+
+  private getDevConfig(): ApmAgentConfig {
+    try {
+      const apmDevConfigPath = join(this.rootDir, 'config', 'apm.dev.js');
+      return require(apmDevConfigPath);
+    } catch (e) {
+      return {};
+    }
+  }
+
+  private getGitRev() {
+    if (this.isDistributable) {
+      return this.pkgBuild.sha;
+    }
+    try {
+      return execSync('git rev-parse --short HEAD', {
+        encoding: 'utf-8' as BufferEncoding,
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+    } catch (e) {
+      return null;
+    }
+  }
+}
diff --git a/packages/kbn-apm-config-loader/src/config_loader.test.mocks.ts b/packages/kbn-apm-config-loader/src/config_loader.test.mocks.ts
new file mode 100644
index 00000000000000..74b50d9daf6323
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config_loader.test.mocks.ts
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const getConfigurationFilePathsMock = jest.fn();
+jest.doMock('./utils/get_config_file_paths', () => ({
+  getConfigurationFilePaths: getConfigurationFilePathsMock,
+}));
+
+export const getConfigFromFilesMock = jest.fn();
+jest.doMock('./utils/read_config', () => ({
+  getConfigFromFiles: getConfigFromFilesMock,
+}));
+
+export const applyConfigOverridesMock = jest.fn();
+jest.doMock('./utils/apply_config_overrides', () => ({
+  applyConfigOverrides: applyConfigOverridesMock,
+}));
+
+export const ApmConfigurationMock = jest.fn();
+jest.doMock('./config', () => ({
+  ApmConfiguration: ApmConfigurationMock,
+}));
+
+export const resetAllMocks = () => {
+  getConfigurationFilePathsMock.mockReset();
+  getConfigFromFilesMock.mockReset();
+  applyConfigOverridesMock.mockReset();
+  ApmConfigurationMock.mockReset();
+};
diff --git a/packages/kbn-apm-config-loader/src/config_loader.test.ts b/packages/kbn-apm-config-loader/src/config_loader.test.ts
new file mode 100644
index 00000000000000..da59237de231e3
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config_loader.test.ts
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  ApmConfigurationMock,
+  applyConfigOverridesMock,
+  getConfigFromFilesMock,
+  getConfigurationFilePathsMock,
+  resetAllMocks,
+} from './config_loader.test.mocks';
+
+import { loadConfiguration } from './config_loader';
+
+describe('loadConfiguration', () => {
+  const argv = ['some', 'arbitrary', 'args'];
+  const rootDir = '/root/dir';
+  const isDistributable = false;
+
+  afterEach(() => {
+    resetAllMocks();
+  });
+
+  it('calls `getConfigurationFilePaths` with the correct arguments', () => {
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(getConfigurationFilePathsMock).toHaveBeenCalledTimes(1);
+    expect(getConfigurationFilePathsMock).toHaveBeenCalledWith(argv);
+  });
+
+  it('calls `getConfigFromFiles` with the correct arguments', () => {
+    const configPaths = ['/path/to/config', '/path/to/other/config'];
+    getConfigurationFilePathsMock.mockReturnValue(configPaths);
+
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(getConfigFromFilesMock).toHaveBeenCalledTimes(1);
+    expect(getConfigFromFilesMock).toHaveBeenCalledWith(configPaths);
+  });
+
+  it('calls `applyConfigOverrides` with the correct arguments', () => {
+    const config = { server: { uuid: 'uuid' } };
+    getConfigFromFilesMock.mockReturnValue(config);
+
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(applyConfigOverridesMock).toHaveBeenCalledTimes(1);
+    expect(applyConfigOverridesMock).toHaveBeenCalledWith(config, argv);
+  });
+
+  it('creates and return an `ApmConfiguration` instance', () => {
+    const apmInstance = { apmInstance: true };
+    ApmConfigurationMock.mockImplementation(() => apmInstance);
+
+    const config = { server: { uuid: 'uuid' } };
+    getConfigFromFilesMock.mockReturnValue(config);
+
+    const instance = loadConfiguration(argv, rootDir, isDistributable);
+    expect(ApmConfigurationMock).toHaveBeenCalledTimes(1);
+    expect(ApmConfigurationMock).toHaveBeenCalledWith(rootDir, config, isDistributable);
+    expect(instance).toBe(apmInstance);
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/config_loader.ts b/packages/kbn-apm-config-loader/src/config_loader.ts
new file mode 100644
index 00000000000000..edddd445b9b7ab
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/config_loader.ts
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getConfigurationFilePaths, getConfigFromFiles, applyConfigOverrides } from './utils';
+import { ApmConfiguration } from './config';
+
+/**
+ * Load the APM configuration.
+ *
+ * @param argv the `process.argv` arguments
+ * @param rootDir The root directory of kibana (where the sources and the `package.json` file are)
+ * @param production true for production builds, false otherwise
+ */
+export const loadConfiguration = (
+  argv: string[],
+  rootDir: string,
+  isDistributable: boolean
+): ApmConfiguration => {
+  const configPaths = getConfigurationFilePaths(argv);
+  const rawConfiguration = getConfigFromFiles(configPaths);
+  applyConfigOverrides(rawConfiguration, argv);
+  return new ApmConfiguration(rootDir, rawConfiguration, isDistributable);
+};
diff --git a/packages/kbn-apm-config-loader/src/index.ts b/packages/kbn-apm-config-loader/src/index.ts
new file mode 100644
index 00000000000000..0d9c057c7cf896
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/index.ts
@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { loadConfiguration } from './config_loader';
+export type { ApmConfiguration } from './config';
+export type { ApmAgentConfig } from './types';
diff --git a/packages/kbn-apm-config-loader/src/types.ts b/packages/kbn-apm-config-loader/src/types.ts
new file mode 100644
index 00000000000000..172edfe0af009b
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/types.ts
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// There is an (incomplete) `AgentConfigOptions` type declared in node_modules/elastic-apm-node/index.d.ts
+// but it's not exported, and using ts tricks to retrieve the type via Parameters<ApmAgent['start']>[0]
+// causes errors in the generated .d.ts file because of esModuleInterop and the fact that the apm module
+// is just exporting an instance of the `ApmAgent` type.
+export type ApmAgentConfig = Record<string, any>;
diff --git a/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap b/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap
new file mode 100644
index 00000000000000..afdce4e76d3f51
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap
@@ -0,0 +1,108 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`different cwd() resolves relative files based on the cwd 1`] = `
+Object {
+  "abc": Object {
+    "def": "test",
+    "qwe": 1,
+    "zyx": Object {
+      "val": 1,
+    },
+  },
+  "bar": true,
+  "empty_arr": Array [],
+  "foo": 1,
+  "pom": Object {
+    "bom": 3,
+  },
+  "xyz": Array [
+    "1",
+    "2",
+  ],
+}
+`;
+
+exports[`reads and merges multiple yaml files from file system and parses to json 1`] = `
+Object {
+  "abc": Object {
+    "def": "test",
+    "ghi": "test2",
+    "qwe": 2,
+    "zyx": Object {},
+  },
+  "arr": Array [
+    1,
+  ],
+  "bar": true,
+  "baz": "bonkers",
+  "empty_arr": Array [],
+  "foo": 2,
+  "pom": Object {
+    "bom": 3,
+    "mob": 4,
+  },
+  "xyz": Array [
+    "3",
+    "4",
+  ],
+}
+`;
+
+exports[`reads single yaml from file system and parses to json 1`] = `
+Object {
+  "arr": Array [
+    1,
+    2,
+  ],
+  "empty_arr": Array [],
+  "empty_obj": Object {},
+  "obj": Object {
+    "val": 3,
+  },
+  "pid": Object {
+    "arr": Array [
+      1,
+    ],
+    "empty_arr": Array [],
+    "empty_obj": Object {},
+    "enabled": true,
+    "file": "/var/run/kibana.pid",
+    "obj": Object {
+      "val": 3,
+    },
+  },
+}
+`;
+
+exports[`returns a deep object 1`] = `
+Object {
+  "pid": Object {
+    "arr": Array [
+      1,
+      2,
+    ],
+    "empty_arr": Array [],
+    "empty_obj": Object {},
+    "enabled": true,
+    "file": "/var/run/kibana.pid",
+    "obj": Object {
+      "val": 3,
+    },
+  },
+}
+`;
+
+exports[`should inject an environment variable value when setting a value with \${ENV_VAR} 1`] = `
+Object {
+  "bar": "pre-val1-mid-val2-post",
+  "elasticsearch": Object {
+    "requestHeadersWhitelist": Array [
+      "val1",
+      "val2",
+    ],
+  },
+  "foo": 1,
+}
+`;
+
+exports[`should throw an exception when referenced environment variable in a config value does not exist 1`] = `"Unknown environment variable referenced in config : KBN_ENV_VAR1"`;
diff --git a/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts b/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts
new file mode 100644
index 00000000000000..1d86f7e1f6e8a2
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { applyConfigOverrides } from './apply_config_overrides';
+
+describe('applyConfigOverrides', () => {
+  it('overrides `server.uuid` when provided as a command line argument', () => {
+    const config: Record<string, any> = {
+      server: {
+        uuid: 'from-config',
+      },
+    };
+    const argv = ['--server.uuid', 'from-argv'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.server.uuid).toEqual('from-argv');
+  });
+
+  it('overrides `path.data` when provided as a command line argument', () => {
+    const config: Record<string, any> = {
+      path: {
+        data: '/from/config',
+      },
+    };
+    const argv = ['--path.data', '/from/argv'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.path.data).toEqual('/from/argv');
+  });
+
+  it('properly set the overridden properties even if the parent object is not present in the config', () => {
+    const config: Record<string, any> = {};
+    const argv = ['--server.uuid', 'from-argv', '--path.data', '/data-path'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.server.uuid).toEqual('from-argv');
+    expect(config.path.data).toEqual('/data-path');
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.ts b/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.ts
new file mode 100644
index 00000000000000..6a3bf95f9954d1
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/apply_config_overrides.ts
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { set } from '@elastic/safer-lodash-set';
+import { getArgValue } from './read_argv';
+
+/**
+ * Manually applies the specific configuration overrides we need to load the APM config.
+ * Currently, only these are needed:
+ *   - server.uuid
+ *   - path.data
+ */
+export const applyConfigOverrides = (config: Record<string, any>, argv: string[]) => {
+  const serverUuid = getArgValue(argv, '--server.uuid');
+  if (serverUuid) {
+    set(config, 'server.uuid', serverUuid);
+  }
+  const dataPath = getArgValue(argv, '--path.data');
+  if (dataPath) {
+    set(config, 'path.data', dataPath);
+  }
+};
diff --git a/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.test.ts b/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.test.ts
new file mode 100644
index 00000000000000..5a520fbeef3169
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.test.ts
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ensureDeepObject } from './ensure_deep_object';
+
+test('flat object', () => {
+  const obj = {
+    'foo.a': 1,
+    'foo.b': 2,
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  });
+});
+
+test('deep object', () => {
+  const obj = {
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  });
+});
+
+test('flat within deep object', () => {
+  const obj = {
+    foo: {
+      b: 2,
+      'bar.a': 1,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      b: 2,
+      bar: {
+        a: 1,
+      },
+    },
+  });
+});
+
+test('flat then flat object', () => {
+  const obj = {
+    'foo.bar': {
+      b: 2,
+      'quux.a': 1,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      bar: {
+        b: 2,
+        quux: {
+          a: 1,
+        },
+      },
+    },
+  });
+});
+
+test('full with empty array', () => {
+  const obj = {
+    a: 1,
+    b: [],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [],
+  });
+});
+
+test('full with array of primitive values', () => {
+  const obj = {
+    a: 1,
+    b: [1, 2, 3],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [1, 2, 3],
+  });
+});
+
+test('full with array of full objects', () => {
+  const obj = {
+    a: 1,
+    b: [{ c: 2 }, { d: 3 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [{ c: 2 }, { d: 3 }],
+  });
+});
+
+test('full with array of flat objects', () => {
+  const obj = {
+    a: 1,
+    b: [{ 'c.d': 2 }, { 'e.f': 3 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [{ c: { d: 2 } }, { e: { f: 3 } }],
+  });
+});
+
+test('flat with flat and array of flat objects', () => {
+  const obj = {
+    a: 1,
+    'b.c': 2,
+    d: [3, { 'e.f': 4 }, { 'g.h': 5 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: { c: 2 },
+    d: [3, { e: { f: 4 } }, { g: { h: 5 } }],
+  });
+});
+
+test('array composed of flat objects', () => {
+  const arr = [{ 'c.d': 2 }, { 'e.f': 3 }];
+
+  expect(ensureDeepObject(arr)).toEqual([{ c: { d: 2 } }, { e: { f: 3 } }]);
+});
diff --git a/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts b/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts
new file mode 100644
index 00000000000000..6eaaef983355c6
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const separator = '.';
+
+/**
+ * Recursively traverses through the object's properties and expands ones with
+ * dot-separated names into nested objects (eg. { a.b: 'c'} -> { a: { b: 'c' }).
+ * @param obj Object to traverse through.
+ * @returns Same object instance with expanded properties.
+ */
+export function ensureDeepObject(obj: any): any {
+  if (obj == null || typeof obj !== 'object') {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => ensureDeepObject(item));
+  }
+
+  return Object.keys(obj).reduce((fullObject, propertyKey) => {
+    const propertyValue = obj[propertyKey];
+    if (!propertyKey.includes(separator)) {
+      fullObject[propertyKey] = ensureDeepObject(propertyValue);
+    } else {
+      walk(fullObject, propertyKey.split(separator), propertyValue);
+    }
+
+    return fullObject;
+  }, {} as any);
+}
+
+function walk(obj: any, keys: string[], value: any) {
+  const key = keys.shift()!;
+  if (keys.length === 0) {
+    obj[key] = value;
+    return;
+  }
+
+  if (obj[key] === undefined) {
+    obj[key] = {};
+  }
+
+  walk(obj[key], keys, ensureDeepObject(value));
+}
diff --git a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts
new file mode 100644
index 00000000000000..c18069f21180bf
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve, join } from 'path';
+import { getConfigPath } from '@kbn/utils';
+import { getConfigurationFilePaths } from './get_config_file_paths';
+
+describe('getConfigurationFilePaths', () => {
+  const cwd = process.cwd();
+
+  it('retrieve the config file paths from the command line arguments', () => {
+    const argv = ['--config', './relative-path', '-c', '/absolute-path'];
+
+    expect(getConfigurationFilePaths(argv)).toEqual([
+      resolve(cwd, join('.', 'relative-path')),
+      '/absolute-path',
+    ]);
+  });
+
+  it('fallbacks to `getConfigPath` value', () => {
+    expect(getConfigurationFilePaths([])).toEqual([getConfigPath()]);
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts
new file mode 100644
index 00000000000000..262f0d1c8b3f51
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+// deep import to avoid loading the whole package
+import { getConfigPath } from '@kbn/utils/target/path';
+import { getArgValues } from './read_argv';
+
+/**
+ * Return the configuration files that needs to be loaded.
+ *
+ * This mimics the behavior of the `src/cli/serve/serve.js` cli script by reading
+ * `-c` and `--config` options from process.argv, and fallbacks to `@kbn/utils`'s `getConfigPath()`
+ */
+export const getConfigurationFilePaths = (argv: string[]): string[] => {
+  const rawPaths = getArgValues(argv, ['-c', '--config']);
+  if (rawPaths.length) {
+    return rawPaths.map((path) => resolve(process.cwd(), path));
+  }
+  return [getConfigPath()];
+};
diff --git a/packages/kbn-apm-config-loader/src/utils/index.ts b/packages/kbn-apm-config-loader/src/utils/index.ts
new file mode 100644
index 00000000000000..03a44e31a44d54
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/index.ts
@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { getConfigFromFiles } from './read_config';
+export { getConfigurationFilePaths } from './get_config_file_paths';
+export { applyConfigOverrides } from './apply_config_overrides';
diff --git a/packages/kbn-apm-config-loader/src/utils/read_argv.test.ts b/packages/kbn-apm-config-loader/src/utils/read_argv.test.ts
new file mode 100644
index 00000000000000..282810e71681e9
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/read_argv.test.ts
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getArgValue, getArgValues } from './read_argv';
+
+describe('getArgValues', () => {
+  it('retrieve the arg value from the provided argv arguments', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      '--config'
+    );
+    expect(argValues).toEqual(['my-config', 'other-config']);
+  });
+
+  it('accept aliases', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual(['my-config', 'other-config']);
+  });
+
+  it('returns an empty array when the arg is not found', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      '--unicorn'
+    );
+    expect(argValues).toEqual([]);
+  });
+
+  it('ignores the flag when no value is provided', () => {
+    const argValues = getArgValues(
+      ['-c', 'my-config', '--foo', '-b', 'bar', '--config'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual(['my-config']);
+  });
+});
+
+describe('getArgValue', () => {
+  it('retrieve the first arg value from the provided argv arguments', () => {
+    const argValues = getArgValue(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      '--config'
+    );
+    expect(argValues).toEqual('my-config');
+  });
+
+  it('accept aliases', () => {
+    const argValues = getArgValue(
+      ['-c', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual('my-config');
+  });
+
+  it('returns undefined the arg is not found', () => {
+    const argValues = getArgValue(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      '--unicorn'
+    );
+    expect(argValues).toBeUndefined();
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/utils/read_argv.ts b/packages/kbn-apm-config-loader/src/utils/read_argv.ts
new file mode 100644
index 00000000000000..9a74d5344a0fc6
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/read_argv.ts
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const getArgValues = (argv: string[], flag: string | string[]): string[] => {
+  const flags = typeof flag === 'string' ? [flag] : flag;
+  const values: string[] = [];
+  for (let i = 0; i < argv.length; i++) {
+    if (flags.includes(argv[i]) && argv[i + 1]) {
+      values.push(argv[++i]);
+    }
+  }
+  return values;
+};
+
+export const getArgValue = (argv: string[], flag: string | string[]): string | undefined => {
+  const values = getArgValues(argv, flag);
+  if (values.length) {
+    return values[0];
+  }
+};
diff --git a/packages/kbn-apm-config-loader/src/utils/read_config.test.ts b/packages/kbn-apm-config-loader/src/utils/read_config.test.ts
new file mode 100644
index 00000000000000..7320e5dcbd6cea
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/read_config.test.ts
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { relative, resolve } from 'path';
+import { getConfigFromFiles } from './read_config';
+
+const fixtureFile = (name: string) => resolve(__dirname, '..', '..', '__fixtures__', name);
+
+test('reads single yaml from file system and parses to json', () => {
+  const config = getConfigFromFiles([fixtureFile('config.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('returns a deep object', () => {
+  const config = getConfigFromFiles([fixtureFile('config_flat.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('reads and merges multiple yaml files from file system and parses to json', () => {
+  const config = getConfigFromFiles([fixtureFile('one.yml'), fixtureFile('two.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('should inject an environment variable value when setting a value with ${ENV_VAR}', () => {
+  process.env.KBN_ENV_VAR1 = 'val1';
+  process.env.KBN_ENV_VAR2 = 'val2';
+
+  const config = getConfigFromFiles([fixtureFile('en_var_ref_config.yml')]);
+
+  delete process.env.KBN_ENV_VAR1;
+  delete process.env.KBN_ENV_VAR2;
+
+  expect(config).toMatchSnapshot();
+});
+
+test('should throw an exception when referenced environment variable in a config value does not exist', () => {
+  expect(() =>
+    getConfigFromFiles([fixtureFile('en_var_ref_config.yml')])
+  ).toThrowErrorMatchingSnapshot();
+});
+
+describe('different cwd()', () => {
+  const originalCwd = process.cwd();
+  const tempCwd = resolve(__dirname);
+
+  beforeAll(() => process.chdir(tempCwd));
+  afterAll(() => process.chdir(originalCwd));
+
+  test('resolves relative files based on the cwd', () => {
+    const relativePath = relative(tempCwd, fixtureFile('one.yml'));
+    const config = getConfigFromFiles([relativePath]);
+
+    expect(config).toMatchSnapshot();
+  });
+
+  test('fails to load relative paths, not found because of the cwd', () => {
+    const relativePath = relative(resolve(__dirname, '..', '..'), fixtureFile('one.yml'));
+    expect(() => getConfigFromFiles([relativePath])).toThrowError(/ENOENT/);
+  });
+});
diff --git a/packages/kbn-apm-config-loader/src/utils/read_config.ts b/packages/kbn-apm-config-loader/src/utils/read_config.ts
new file mode 100644
index 00000000000000..825bfd60181bf2
--- /dev/null
+++ b/packages/kbn-apm-config-loader/src/utils/read_config.ts
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readFileSync } from 'fs';
+import { safeLoad } from 'js-yaml';
+
+import { set } from '@elastic/safer-lodash-set';
+import { isPlainObject } from 'lodash';
+import { ensureDeepObject } from './ensure_deep_object';
+
+const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8'));
+
+function replaceEnvVarRefs(val: string) {
+  return val.replace(/\$\{(\w+)\}/g, (match, envVarName) => {
+    const envVarValue = process.env[envVarName];
+    if (envVarValue !== undefined) {
+      return envVarValue;
+    }
+
+    throw new Error(`Unknown environment variable referenced in config : ${envVarName}`);
+  });
+}
+
+function merge(target: Record<string, any>, value: any, key?: string) {
+  if ((isPlainObject(value) || Array.isArray(value)) && Object.keys(value).length > 0) {
+    for (const [subKey, subVal] of Object.entries(value)) {
+      merge(target, subVal, key ? `${key}.${subKey}` : subKey);
+    }
+  } else if (key !== undefined) {
+    set(target, key, typeof value === 'string' ? replaceEnvVarRefs(value) : value);
+  }
+
+  return target;
+}
+
+/** @internal */
+export const getConfigFromFiles = (configFiles: readonly string[]): Record<string, any> => {
+  let mergedYaml: Record<string, any> = {};
+
+  for (const configFile of configFiles) {
+    const yaml = readYaml(configFile);
+    if (yaml !== null) {
+      mergedYaml = merge(mergedYaml, yaml);
+    }
+  }
+
+  return ensureDeepObject(mergedYaml);
+};
diff --git a/packages/kbn-apm-config-loader/tsconfig.json b/packages/kbn-apm-config-loader/tsconfig.json
new file mode 100644
index 00000000000000..ba00ddfa6adb6f
--- /dev/null
+++ b/packages/kbn-apm-config-loader/tsconfig.json
@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "declaration": true,
+    "outDir": "./target",
+    "stripInternal": false,
+    "declarationMap": true,
+    "types": ["jest", "node"]
+  },
+  "include": ["./src/**/*.ts"],
+  "exclude": ["target"]
+}
diff --git a/packages/kbn-apm-config-loader/yarn.lock b/packages/kbn-apm-config-loader/yarn.lock
new file mode 120000
index 00000000000000..3f82ebc9cdbae3
--- /dev/null
+++ b/packages/kbn-apm-config-loader/yarn.lock
@@ -0,0 +1 @@
+../../yarn.lock
\ No newline at end of file
diff --git a/src/apm.js b/src/apm.js
index effa6c77d76148..8a0c010d993f14 100644
--- a/src/apm.js
+++ b/src/apm.js
@@ -18,67 +18,11 @@
  */
 
 const { join } = require('path');
-const { readFileSync } = require('fs');
-const { execSync } = require('child_process');
-const { merge } = require('lodash');
-const { name, version, build } = require('../package.json');
+const { name, build } = require('../package.json');
+const { loadConfiguration } = require('@kbn/apm-config-loader');
 
 const ROOT_DIR = join(__dirname, '..');
-
-function gitRev() {
-  try {
-    return execSync('git rev-parse --short HEAD', {
-      encoding: 'utf-8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim();
-  } catch (e) {
-    return null;
-  }
-}
-
-function devConfig() {
-  try {
-    const apmDevConfigPath = join(ROOT_DIR, 'config', 'apm.dev.js');
-    return require(apmDevConfigPath); // eslint-disable-line import/no-dynamic-require
-  } catch (e) {
-    return {};
-  }
-}
-
-const apmConfig = merge(
-  {
-    active: false,
-    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
-    // The secretToken below is intended to be hardcoded in this file even though
-    // it makes it public. This is not a security/privacy issue. Normally we'd
-    // instead disable the need for a secretToken in the APM Server config where
-    // the data is transmitted to, but due to how it's being hosted, it's easier,
-    // for now, to simply leave it in.
-    secretToken: 'R0Gjg46pE9K9wGestd',
-    globalLabels: {},
-    breakdownMetrics: true,
-    centralConfig: false,
-    logUncaughtExceptions: true,
-  },
-  devConfig()
-);
-
-try {
-  const filename = join(ROOT_DIR, 'data', 'uuid');
-  apmConfig.globalLabels.kibana_uuid = readFileSync(filename, 'utf-8');
-} catch (e) {} // eslint-disable-line no-empty
-
-const rev = gitRev();
-if (rev !== null) apmConfig.globalLabels.git_rev = rev;
-
-function getConfig(serviceName) {
-  return {
-    ...apmConfig,
-    ...{
-      serviceName: `${serviceName}-${version.replace(/\./g, '_')}`,
-    },
-  };
-}
+let apmConfig;
 
 /**
  * Flag to disable APM RUM support on all kibana builds by default
@@ -86,12 +30,24 @@ function getConfig(serviceName) {
 const isKibanaDistributable = Boolean(build && build.distributable === true);
 
 module.exports = function (serviceName = name) {
-  if (process.env.kbnWorkerType === 'optmzr') return;
-
-  const conf = getConfig(serviceName);
+  if (process.env.kbnWorkerType === 'optmzr') {
+    return;
+  }
 
+  apmConfig = loadConfiguration(process.argv, ROOT_DIR, isKibanaDistributable);
+  const conf = apmConfig.getConfig(serviceName);
   require('elastic-apm-node').start(conf);
 };
 
-module.exports.getConfig = getConfig;
+module.exports.getConfig = (serviceName) => {
+  // integration test runner starts a kibana server that import the module without initializing APM.
+  // so we need to check initialization of the config.
+  // note that we can't just load the configuration during this module's import
+  // because jest IT are ran with `--config path-to-jest-config.js` which conflicts with the CLI's `config` arg
+  // causing the config loader to try to load the jest js config as yaml and throws.
+  if (apmConfig) {
+    return apmConfig.getConfig(serviceName);
+  }
+  return {};
+};
 module.exports.isKibanaDistributable = isKibanaDistributable;