[Metrics UI] Anomaly Detection setup flow for Metrics (elastic#76787)

* adds metrics ml integration

* Add ability to create ml jobs from inventory

* Fix i18n stuff

* Fix typecheck

* renames jobs, updates datafeeds

* adds allow_no_indices: true for datafeeds

* Revert "[Metrics UI] Replace Snapshot API with Metrics API (elastic#76253)"

This reverts commit 0ca6472.

* Add ability to fetch anomalies

* Fix typecheck

* Fix typecheck

* Fix i18n

* Fix lint, use the right partition field

* Delete log files

* Fix merge

* Fix merge issues

* Update name of jobs

* Remove CPU job

* [Metrics UI] Replace Snapshot API with Metrics API (elastic#76253)

- Remove server/lib/snapshot
- Replace backend for /api/infra/snapshot with data from Metrics API
- Fixing tests with updates to the snapshot node

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

* Add links back to ML for anomalies and manage jobs

* Fix typecheck

* Remove unecessary validation

Co-authored-by: Michael Hirsch <michaelahirsch@gmail.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Chris Cowan <chris@chriscowan.us>

x-pack/plugins/infra/common/http_api/infra_ml/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './results';

x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+// [Sort field value, tiebreaker value]
+export const paginationCursorRT = rt.tuple([
+ rt.union([rt.string, rt.number]),
+ rt.union([rt.string, rt.number]),
+]);
+
+export type PaginationCursor = rt.TypeOf<typeof paginationCursorRT>;
+
+export const anomalyTypeRT = rt.keyof({
+ metrics_hosts: null,
+ metrics_k8s: null,
+});
+
+export type AnomalyType = rt.TypeOf<typeof anomalyTypeRT>;
+
+const sortOptionsRT = rt.keyof({
+ anomalyScore: null,
+ dataset: null,
+ startTime: null,
+});
+
+const sortDirectionsRT = rt.keyof({
+ asc: null,
+ desc: null,
+});
+
+const paginationPreviousPageCursorRT = rt.type({
+ searchBefore: paginationCursorRT,
+});
+
+const paginationNextPageCursorRT = rt.type({
+ searchAfter: paginationCursorRT,
+});
+
+export const paginationRT = rt.intersection([
+ rt.type({
+ pageSize: rt.number,
+ }),
+ rt.partial({
+ cursor: rt.union([paginationPreviousPageCursorRT, paginationNextPageCursorRT]),
+ }),
+]);
+
+export type Pagination = rt.TypeOf<typeof paginationRT>;
+
+export const sortRT = rt.type({
+ field: sortOptionsRT,
+ direction: sortDirectionsRT,
+});
+
+export type Sort = rt.TypeOf<typeof sortRT>;

x-pack/plugins/infra/common/http_api/infra_ml/results/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './metrics_hosts_anomalies';
+export * from './metrics_k8s_anomalies';
+export * from './common';

x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
+import { anomalyTypeRT, paginationCursorRT, sortRT, paginationRT } from './common';
+
+export const INFA_ML_GET_METRICS_HOSTS_ANOMALIES_PATH =
+ '/api/infra/infra_ml/results/metrics_hosts_anomalies';
+
+const metricsHostAnomalyCommonFieldsRT = rt.type({
+ id: rt.string,
+ anomalyScore: rt.number,
+ typical: rt.number,
+ actual: rt.number,
+ type: anomalyTypeRT,
+ duration: rt.number,
+ startTime: rt.number,
+ jobId: rt.string,
+});
+const metricsHostsAnomalyRT = metricsHostAnomalyCommonFieldsRT;
+
+export type MetricsHostsAnomaly = rt.TypeOf<typeof metricsHostsAnomalyRT>;
+
+export const getMetricsHostsAnomaliesSuccessReponsePayloadRT = rt.intersection([
+ rt.type({
+ data: rt.intersection([
+ rt.type({
+ anomalies: rt.array(metricsHostsAnomalyRT),
+ // Signifies there are more entries backwards or forwards. If this was a request
+ // for a previous page, there are more previous pages, if this was a request for a next page,
+ // there are more next pages.
+ hasMoreEntries: rt.boolean,
+ }),
+ rt.partial({
+ paginationCursors: rt.type({
+ // The cursor to use to fetch the previous page
+ previousPageCursor: paginationCursorRT,
+ // The cursor to use to fetch the next page
+ nextPageCursor: paginationCursorRT,
+ }),
+ }),
+ ]),
+ }),
+ rt.partial({
+ timing: routeTimingMetadataRT,
+ }),
+]);
+
+export type GetMetricsHostsAnomaliesSuccessResponsePayload = rt.TypeOf<
+ typeof getMetricsHostsAnomaliesSuccessReponsePayloadRT
+>;
+
+export const getMetricsHostsAnomaliesRequestPayloadRT = rt.type({
+ data: rt.intersection([
+ rt.type({
+ // the ID of the source configuration
+ sourceId: rt.string,
+ // the time range to fetch the log entry anomalies from
+ timeRange: timeRangeRT,
+ }),
+ rt.partial({
+ // Pagination properties
+ pagination: paginationRT,
+ // Sort properties
+ sort: sortRT,
+ // // Dataset filters
+ // datasets: rt.array(rt.string),
+ }),
+ ]),
+});
+
+export type GetMetricsHostsAnomaliesRequestPayload = rt.TypeOf<
+ typeof getMetricsHostsAnomaliesRequestPayloadRT
+>;

x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
+import { paginationCursorRT, anomalyTypeRT, sortRT, paginationRT } from './common';
+
+export const INFA_ML_GET_METRICS_K8S_ANOMALIES_PATH =
+ '/api/infra/infra_ml/results/metrics_k8s_anomalies';
+
+const metricsK8sAnomalyCommonFieldsRT = rt.type({
+ id: rt.string,
+ anomalyScore: rt.number,
+ typical: rt.number,
+ actual: rt.number,
+ type: anomalyTypeRT,
+ duration: rt.number,
+ startTime: rt.number,
+ jobId: rt.string,
+});
+const metricsK8sAnomalyRT = metricsK8sAnomalyCommonFieldsRT;
+
+export type MetricsK8sAnomaly = rt.TypeOf<typeof metricsK8sAnomalyRT>;
+
+export const getMetricsK8sAnomaliesSuccessReponsePayloadRT = rt.intersection([
+ rt.type({
+ data: rt.intersection([
+ rt.type({
+ anomalies: rt.array(metricsK8sAnomalyRT),
+ // Signifies there are more entries backwards or forwards. If this was a request
+ // for a previous page, there are more previous pages, if this was a request for a next page,
+ // there are more next pages.
+ hasMoreEntries: rt.boolean,
+ }),
+ rt.partial({
+ paginationCursors: rt.type({
+ // The cursor to use to fetch the previous page
+ previousPageCursor: paginationCursorRT,
+ // The cursor to use to fetch the next page
+ nextPageCursor: paginationCursorRT,
+ }),
+ }),
+ ]),
+ }),
+ rt.partial({
+ timing: routeTimingMetadataRT,
+ }),
+]);
+
+export type GetMetricsK8sAnomaliesSuccessResponsePayload = rt.TypeOf<
+ typeof getMetricsK8sAnomaliesSuccessReponsePayloadRT
+>;
+
+export const getMetricsK8sAnomaliesRequestPayloadRT = rt.type({
+ data: rt.intersection([
+ rt.type({
+ // the ID of the source configuration
+ sourceId: rt.string,
+ // the time range to fetch the log entry anomalies from
+ timeRange: timeRangeRT,
+ }),
+ rt.partial({
+ // Pagination properties
+ pagination: paginationRT,
+ // Sort properties
+ sort: sortRT,
+ // Dataset filters
+ datasets: rt.array(rt.string),
+ }),
+ ]),
+});
+
+export type GetMetricsK8sAnomaliesRequestPayload = rt.TypeOf<
+ typeof getMetricsK8sAnomaliesRequestPayloadRT
+>;

x-pack/plugins/infra/common/infra_ml/anomaly_results.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const ML_SEVERITY_SCORES = {
+ warning: 3,
+ minor: 25,
+ major: 50,
+ critical: 75,
+};
+
+export type MLSeverityScoreCategories = keyof typeof ML_SEVERITY_SCORES;
+
+export const ML_SEVERITY_COLORS = {
+ critical: 'rgb(228, 72, 72)',
+ major: 'rgb(229, 113, 0)',
+ minor: 'rgb(255, 221, 0)',
+ warning: 'rgb(125, 180, 226)',
+};
+
+export const getSeverityCategoryForScore = (
+ score: number
+): MLSeverityScoreCategories | undefined => {
+ if (score >= ML_SEVERITY_SCORES.critical) {
+ return 'critical';
+ } else if (score >= ML_SEVERITY_SCORES.major) {
+ return 'major';
+ } else if (score >= ML_SEVERITY_SCORES.minor) {
+ return 'minor';
+ } else if (score >= ML_SEVERITY_SCORES.warning) {
+ return 'warning';
+ } else {
+ // Category is too low to include
+ return undefined;
+ }
+};
+
+export const formatAnomalyScore = (score: number) => {
+ return Math.round(score);
+};
+
+export const formatOneDecimalPlace = (number: number) => {
+ return Math.round(number * 10) / 10;
+};
+
+export const getFriendlyNameForPartitionId = (partitionId: string) => {
+ return partitionId !== '' ? partitionId : 'unknown';
+};
+
+export const compareDatasetsByMaximumAnomalyScore = <
+ Dataset extends { maximumAnomalyScore: number }
+>(
+ firstDataset: Dataset,
+ secondDataset: Dataset
+) => firstDataset.maximumAnomalyScore - secondDataset.maximumAnomalyScore;

x-pack/plugins/infra/common/infra_ml/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './infra_ml';
+export * from './anomaly_results';
+export * from './job_parameters';
+export * from './metrics_hosts_ml';
+export * from './metrics_k8s_ml';

x-pack/plugins/infra/common/infra_ml/infra_ml.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// combines and abstracts job and datafeed status
+export type JobStatus =
+ | 'unknown'
+ | 'missing'
+ | 'initializing'
+ | 'stopped'
+ | 'started'
+ | 'finished'
+ | 'failed';
+
+export type SetupStatus =
+ | { type: 'initializing' } // acquiring job statuses to determine setup status
+ | { type: 'unknown' } // job status could not be acquired (failed request etc)
+ | { type: 'required' } // setup required
+ | { type: 'pending' } // In the process of setting up the module for the first time or retrying, waiting for response
+ | { type: 'succeeded' } // setup succeeded, notifying user
+ | {
+ type: 'failed';
+ reasons: string[];
+ } // setup failed, notifying user
+ | {
+ type: 'skipped';
+ newlyCreated?: boolean;
+ }; // setup is not necessary
+
+/**
+ * Maps a job status to the possibility that results have already been produced
+ * before this state was reached.
+ */
+export const isJobStatusWithResults = (jobStatus: JobStatus) =>
+ ['started', 'finished', 'stopped', 'failed'].includes(jobStatus);
+
+export const isHealthyJobStatus = (jobStatus: JobStatus) =>
+ ['started', 'finished'].includes(jobStatus);
+
+/**
+ * Maps a setup status to the possibility that results have already been
+ * produced before this state was reached.
+ */
+export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
+ setupStatus.type === 'skipped';
+
+const KIBANA_SAMPLE_DATA_INDICES = ['kibana_sample_data_logs*'];
+
+export const isExampleDataIndex = (indexName: string) =>
+ KIBANA_SAMPLE_DATA_INDICES.includes(indexName);