[Metrics UI] Anomaly Detection setup flow for Metrics (#76787) (#78432)

* adds metrics ml integration

* Add ability to create ml jobs from inventory

* Fix i18n stuff

* Fix typecheck

* renames jobs, updates datafeeds

* adds allow_no_indices: true for datafeeds

* Revert "[Metrics UI] Replace Snapshot API with Metrics API (#76253)"

This reverts commit 0ca6472.

* Add ability to fetch anomalies

* Fix typecheck

* Fix typecheck

* Fix i18n

* Fix lint, use the right partition field

* Delete log files

* Fix merge

* Fix merge issues

* Update name of jobs

* Remove CPU job

* [Metrics UI] Replace Snapshot API with Metrics API (#76253)

- Remove server/lib/snapshot
- Replace backend for /api/infra/snapshot with data from Metrics API
- Fixing tests with updates to the snapshot node

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

* Add links back to ML for anomalies and manage jobs

* Fix typecheck

* Remove unecessary validation

Co-authored-by: Michael Hirsch <michaelahirsch@gmail.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Chris Cowan <chris@chriscowan.us>

Co-authored-by: Michael Hirsch <michaelahirsch@gmail.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Chris Cowan <chris@chriscowan.us>

x-pack/plugins/infra/common/http_api/infra_ml/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './results';

x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+// [Sort field value, tiebreaker value]
+export const paginationCursorRT = rt.tuple([
+  rt.union([rt.string, rt.number]),
+  rt.union([rt.string, rt.number]),
+]);
+
+export type PaginationCursor = rt.TypeOf<typeof paginationCursorRT>;
+
+export const anomalyTypeRT = rt.keyof({
+  metrics_hosts: null,
+  metrics_k8s: null,
+});
+
+export type AnomalyType = rt.TypeOf<typeof anomalyTypeRT>;
+
+const sortOptionsRT = rt.keyof({
+  anomalyScore: null,
+  dataset: null,
+  startTime: null,
+});
+
+const sortDirectionsRT = rt.keyof({
+  asc: null,
+  desc: null,
+});
+
+const paginationPreviousPageCursorRT = rt.type({
+  searchBefore: paginationCursorRT,
+});
+
+const paginationNextPageCursorRT = rt.type({
+  searchAfter: paginationCursorRT,
+});
+
+export const paginationRT = rt.intersection([
+  rt.type({
+    pageSize: rt.number,
+  }),
+  rt.partial({
+    cursor: rt.union([paginationPreviousPageCursorRT, paginationNextPageCursorRT]),
+  }),
+]);
+
+export type Pagination = rt.TypeOf<typeof paginationRT>;
+
+export const sortRT = rt.type({
+  field: sortOptionsRT,
+  direction: sortDirectionsRT,
+});
+
+export type Sort = rt.TypeOf<typeof sortRT>;

x-pack/plugins/infra/common/http_api/infra_ml/results/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './metrics_hosts_anomalies';
+export * from './metrics_k8s_anomalies';
+export * from './common';

x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
+import { anomalyTypeRT, paginationCursorRT, sortRT, paginationRT } from './common';
+
+export const INFA_ML_GET_METRICS_HOSTS_ANOMALIES_PATH =
+  '/api/infra/infra_ml/results/metrics_hosts_anomalies';
+
+const metricsHostAnomalyCommonFieldsRT = rt.type({
+  id: rt.string,
+  anomalyScore: rt.number,
+  typical: rt.number,
+  actual: rt.number,
+  type: anomalyTypeRT,
+  duration: rt.number,
+  startTime: rt.number,
+  jobId: rt.string,
+});
+const metricsHostsAnomalyRT = metricsHostAnomalyCommonFieldsRT;
+
+export type MetricsHostsAnomaly = rt.TypeOf<typeof metricsHostsAnomalyRT>;
+
+export const getMetricsHostsAnomaliesSuccessReponsePayloadRT = rt.intersection([
+  rt.type({
+    data: rt.intersection([
+      rt.type({
+        anomalies: rt.array(metricsHostsAnomalyRT),
+        // Signifies there are more entries backwards or forwards. If this was a request
+        // for a previous page, there are more previous pages, if this was a request for a next page,
+        // there are more next pages.
+        hasMoreEntries: rt.boolean,
+      }),
+      rt.partial({
+        paginationCursors: rt.type({
+          // The cursor to use to fetch the previous page
+          previousPageCursor: paginationCursorRT,
+          // The cursor to use to fetch the next page
+          nextPageCursor: paginationCursorRT,
+        }),
+      }),
+    ]),
+  }),
+  rt.partial({
+    timing: routeTimingMetadataRT,
+  }),
+]);
+
+export type GetMetricsHostsAnomaliesSuccessResponsePayload = rt.TypeOf<
+  typeof getMetricsHostsAnomaliesSuccessReponsePayloadRT
+>;
+
+export const getMetricsHostsAnomaliesRequestPayloadRT = rt.type({
+  data: rt.intersection([
+    rt.type({
+      // the ID of the source configuration
+      sourceId: rt.string,
+      // the time range to fetch the log entry anomalies from
+      timeRange: timeRangeRT,
+    }),
+    rt.partial({
+      // Pagination properties
+      pagination: paginationRT,
+      // Sort properties
+      sort: sortRT,
+      // // Dataset filters
+      // datasets: rt.array(rt.string),
+    }),
+  ]),
+});
+
+export type GetMetricsHostsAnomaliesRequestPayload = rt.TypeOf<
+  typeof getMetricsHostsAnomaliesRequestPayloadRT
+>;

x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
+import { paginationCursorRT, anomalyTypeRT, sortRT, paginationRT } from './common';
+
+export const INFA_ML_GET_METRICS_K8S_ANOMALIES_PATH =
+  '/api/infra/infra_ml/results/metrics_k8s_anomalies';
+
+const metricsK8sAnomalyCommonFieldsRT = rt.type({
+  id: rt.string,
+  anomalyScore: rt.number,
+  typical: rt.number,
+  actual: rt.number,
+  type: anomalyTypeRT,
+  duration: rt.number,
+  startTime: rt.number,
+  jobId: rt.string,
+});
+const metricsK8sAnomalyRT = metricsK8sAnomalyCommonFieldsRT;
+
+export type MetricsK8sAnomaly = rt.TypeOf<typeof metricsK8sAnomalyRT>;
+
+export const getMetricsK8sAnomaliesSuccessReponsePayloadRT = rt.intersection([
+  rt.type({
+    data: rt.intersection([
+      rt.type({
+        anomalies: rt.array(metricsK8sAnomalyRT),
+        // Signifies there are more entries backwards or forwards. If this was a request
+        // for a previous page, there are more previous pages, if this was a request for a next page,
+        // there are more next pages.
+        hasMoreEntries: rt.boolean,
+      }),
+      rt.partial({
+        paginationCursors: rt.type({
+          // The cursor to use to fetch the previous page
+          previousPageCursor: paginationCursorRT,
+          // The cursor to use to fetch the next page
+          nextPageCursor: paginationCursorRT,
+        }),
+      }),
+    ]),
+  }),
+  rt.partial({
+    timing: routeTimingMetadataRT,
+  }),
+]);
+
+export type GetMetricsK8sAnomaliesSuccessResponsePayload = rt.TypeOf<
+  typeof getMetricsK8sAnomaliesSuccessReponsePayloadRT
+>;
+
+export const getMetricsK8sAnomaliesRequestPayloadRT = rt.type({
+  data: rt.intersection([
+    rt.type({
+      // the ID of the source configuration
+      sourceId: rt.string,
+      // the time range to fetch the log entry anomalies from
+      timeRange: timeRangeRT,
+    }),
+    rt.partial({
+      // Pagination properties
+      pagination: paginationRT,
+      // Sort properties
+      sort: sortRT,
+      // Dataset filters
+      datasets: rt.array(rt.string),
+    }),
+  ]),
+});
+
+export type GetMetricsK8sAnomaliesRequestPayload = rt.TypeOf<
+  typeof getMetricsK8sAnomaliesRequestPayloadRT
+>;

x-pack/plugins/infra/common/infra_ml/anomaly_results.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const ML_SEVERITY_SCORES = {
+  warning: 3,
+  minor: 25,
+  major: 50,
+  critical: 75,
+};
+
+export type MLSeverityScoreCategories = keyof typeof ML_SEVERITY_SCORES;
+
+export const ML_SEVERITY_COLORS = {
+  critical: 'rgb(228, 72, 72)',
+  major: 'rgb(229, 113, 0)',
+  minor: 'rgb(255, 221, 0)',
+  warning: 'rgb(125, 180, 226)',
+};
+
+export const getSeverityCategoryForScore = (
+  score: number
+): MLSeverityScoreCategories | undefined => {
+  if (score >= ML_SEVERITY_SCORES.critical) {
+    return 'critical';
+  } else if (score >= ML_SEVERITY_SCORES.major) {
+    return 'major';
+  } else if (score >= ML_SEVERITY_SCORES.minor) {
+    return 'minor';
+  } else if (score >= ML_SEVERITY_SCORES.warning) {
+    return 'warning';
+  } else {
+    // Category is too low to include
+    return undefined;
+  }
+};
+
+export const formatAnomalyScore = (score: number) => {
+  return Math.round(score);
+};
+
+export const formatOneDecimalPlace = (number: number) => {
+  return Math.round(number * 10) / 10;
+};
+
+export const getFriendlyNameForPartitionId = (partitionId: string) => {
+  return partitionId !== '' ? partitionId : 'unknown';
+};
+
+export const compareDatasetsByMaximumAnomalyScore = <
+  Dataset extends { maximumAnomalyScore: number }
+>(
+  firstDataset: Dataset,
+  secondDataset: Dataset
+) => firstDataset.maximumAnomalyScore - secondDataset.maximumAnomalyScore;

x-pack/plugins/infra/common/infra_ml/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './infra_ml';
+export * from './anomaly_results';
+export * from './job_parameters';
+export * from './metrics_hosts_ml';
+export * from './metrics_k8s_ml';

x-pack/plugins/infra/common/infra_ml/infra_ml.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// combines and abstracts job and datafeed status
+export type JobStatus =
+  | 'unknown'
+  | 'missing'
+  | 'initializing'
+  | 'stopped'
+  | 'started'
+  | 'finished'
+  | 'failed';
+
+export type SetupStatus =
+  | { type: 'initializing' } // acquiring job statuses to determine setup status
+  | { type: 'unknown' } // job status could not be acquired (failed request etc)
+  | { type: 'required' } // setup required
+  | { type: 'pending' } // In the process of setting up the module for the first time or retrying, waiting for response
+  | { type: 'succeeded' } // setup succeeded, notifying user
+  | {
+      type: 'failed';
+      reasons: string[];
+    } // setup failed, notifying user
+  | {
+      type: 'skipped';
+      newlyCreated?: boolean;
+    }; // setup is not necessary
+
+/**
+ * Maps a job status to the possibility that results have already been produced
+ * before this state was reached.
+ */
+export const isJobStatusWithResults = (jobStatus: JobStatus) =>
+  ['started', 'finished', 'stopped', 'failed'].includes(jobStatus);
+
+export const isHealthyJobStatus = (jobStatus: JobStatus) =>
+  ['started', 'finished'].includes(jobStatus);
+
+/**
+ * Maps a setup status to the possibility that results have already been
+ * produced before this state was reached.
+ */
+export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
+  setupStatus.type === 'skipped';
+
+const KIBANA_SAMPLE_DATA_INDICES = ['kibana_sample_data_logs*'];
+
+export const isExampleDataIndex = (indexName: string) =>
+  KIBANA_SAMPLE_DATA_INDICES.includes(indexName);