From 607ee02f9ad699bcf182b80f4af068ed21ca469e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 7 Jul 2020 19:56:28 +0200 Subject: [PATCH] [APM] Adds 'Anomaly detection' settings page to create ML jobs per environment (#70560) (#70945) --- .../app/Main/route_config/index.tsx | 17 ++ .../app/Main/route_config/route_names.tsx | 1 + .../anomaly_detection/add_environments.tsx | 164 ++++++++++++++++++ .../Settings/anomaly_detection/create_jobs.ts | 64 +++++++ .../app/Settings/anomaly_detection/index.tsx | 68 ++++++++ .../Settings/anomaly_detection/jobs_list.tsx | 162 +++++++++++++++++ .../public/components/app/Settings/index.tsx | 23 ++- .../components/shared/ManagedTable/index.tsx | 11 +- .../create_anomaly_detection_jobs.ts | 123 +++++++++++++ .../get_anomaly_detection_jobs.ts | 60 +++++++ .../get_all_environments.test.ts.snap | 85 +++++++++ .../environments/get_all_environments.test.ts | 42 +++++ .../get_all_environments.ts | 13 +- .../apm/server/lib/helpers/setup_request.ts | 5 + .../__snapshots__/queries.test.ts.snap | 41 ----- .../get_environments/index.ts | 5 +- .../agent_configuration/queries.test.ts | 14 -- .../apm/server/routes/create_apm_api.ts | 12 +- .../routes/settings/anomaly_detection.ts | 55 ++++++ .../plugins/apm/typings/anomaly_detection.ts | 10 ++ .../types/anomaly_detection_jobs/job.ts | 3 + 21 files changed, 906 insertions(+), 72 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx create mode 100644 x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts create mode 100644 x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts create mode 100644 x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap create mode 100644 x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts rename x-pack/plugins/apm/server/lib/{settings/agent_configuration/get_environments => environments}/get_all_environments.ts (78%) create mode 100644 x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts create mode 100644 x-pack/plugins/apm/typings/anomaly_detection.ts diff --git a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx index 1625fb4c1409ef..8379def2a7d9aa 100644 --- a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx @@ -23,6 +23,7 @@ import { resolveUrlParams } from '../../../../context/UrlParamsContext/resolveUr import { UNIDENTIFIED_SERVICE_NODES_LABEL } from '../../../../../common/i18n'; import { TraceLink } from '../../TraceLink'; import { CustomizeUI } from '../../Settings/CustomizeUI'; +import { AnomalyDetection } from '../../Settings/anomaly_detection'; import { EditAgentConfigurationRouteHandler, CreateAgentConfigurationRouteHandler, @@ -268,4 +269,20 @@ export const routes: BreadcrumbRoute[] = [ }), name: RouteName.RUM_OVERVIEW, }, + { + exact: true, + path: '/settings/anomaly-detection', + component: () => ( + + + + ), + breadcrumb: i18n.translate( + 'xpack.apm.breadcrumb.settings.anomalyDetection', + { + defaultMessage: 'Anomaly detection', + } + ), + name: RouteName.ANOMALY_DETECTION, + }, ]; diff --git a/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx b/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx index 4965aa9db87602..37d96e74d8ee6e 100644 --- a/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx +++ b/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx @@ -27,4 +27,5 @@ export enum RouteName { LINK_TO_TRACE = 'link_to_trace', CUSTOMIZE_UI = 'customize_ui', RUM_OVERVIEW = 'rum_overview', + ANOMALY_DETECTION = 'anomaly_detection', } diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx new file mode 100644 index 00000000000000..2da3c125631043 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState } from 'react'; +import { + EuiPanel, + EuiTitle, + EuiText, + EuiSpacer, + EuiButton, + EuiButtonEmpty, + EuiComboBox, + EuiComboBoxOptionOption, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useFetcher, FETCH_STATUS } from '../../../../hooks/useFetcher'; +import { useApmPluginContext } from '../../../../hooks/useApmPluginContext'; +import { createJobs } from './create_jobs'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../../../common/environment_filter_values'; + +interface Props { + currentEnvironments: string[]; + onCreateJobSuccess: () => void; + onCancel: () => void; +} +export const AddEnvironments = ({ + currentEnvironments, + onCreateJobSuccess, + onCancel, +}: Props) => { + const { toasts } = useApmPluginContext().core.notifications; + const { data = [], status } = useFetcher( + (callApmApi) => + callApmApi({ + pathname: `/api/apm/settings/anomaly-detection/environments`, + }), + [], + { preservePreviousData: false } + ); + + const environmentOptions = data.map((env) => ({ + label: env === ENVIRONMENT_NOT_DEFINED ? NOT_DEFINED_OPTION_LABEL : env, + value: env, + disabled: currentEnvironments.includes(env), + })); + + const [selectedOptions, setSelected] = useState< + Array> + >([]); + + const isLoading = + status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING; + return ( + + +

+ {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.titleText', + { + defaultMessage: 'Select environments', + } + )} +

+ + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.descriptionText', + { + defaultMessage: + 'Select the service environments that you want to enable anomaly detection in. Anomalies will surface for all services and transaction types within the selected environments.', + } + )} + + + + { + setSelected(nextSelectedOptions); + }} + onCreateOption={(searchValue) => { + if (currentEnvironments.includes(searchValue)) { + return; + } + const newOption = { + label: searchValue, + value: searchValue, + }; + setSelected([...selectedOptions, newOption]); + }} + isClearable={true} + /> + + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.cancelButtonText', + { + defaultMessage: 'Cancel', + } + )} + + + + { + const selectedEnvironments = selectedOptions.map( + ({ value }) => value as string + ); + const success = await createJobs({ + environments: selectedEnvironments, + toasts, + }); + if (success) { + onCreateJobSuccess(); + } + }} + > + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.createJobsButtonText', + { + defaultMessage: 'Create Jobs', + } + )} + + + + + + ); +}; + +const NOT_DEFINED_OPTION_LABEL = i18n.translate( + 'xpack.apm.filter.environment.notDefinedLabel', + { + defaultMessage: 'Not defined', + } +); diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts new file mode 100644 index 00000000000000..614632a5a3b092 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { NotificationsStart } from 'kibana/public'; +import { callApmApi } from '../../../../services/rest/createCallApmApi'; + +export async function createJobs({ + environments, + toasts, +}: { + environments: string[]; + toasts: NotificationsStart['toasts']; +}) { + try { + await callApmApi({ + pathname: '/api/apm/settings/anomaly-detection/jobs', + method: 'POST', + params: { + body: { environments }, + }, + }); + + toasts.addSuccess({ + title: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.succeeded.title', + { defaultMessage: 'Anomaly detection jobs created' } + ), + text: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.succeeded.text', + { + defaultMessage: + 'Anomaly detection jobs successfully created for APM service environments [{environments}]. It will take some time for machine learning to start analyzing traffic for anomalies.', + values: { environments: environments.join(', ') }, + } + ), + }); + return true; + } catch (error) { + toasts.addDanger({ + title: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.failed.title', + { + defaultMessage: 'Anomaly detection jobs could not be created', + } + ), + text: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.failed.text', + { + defaultMessage: + 'Something went wrong when creating one ore more anomaly detection jobs for APM service environments [{environments}]. Error: "{errorMessage}"', + values: { + environments: environments.join(', '), + errorMessage: error.message, + }, + } + ), + }); + return false; + } +} diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx new file mode 100644 index 00000000000000..0b720242237014 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState } from 'react'; +import { EuiTitle, EuiSpacer, EuiText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { JobsList } from './jobs_list'; +import { AddEnvironments } from './add_environments'; +import { useFetcher, FETCH_STATUS } from '../../../../hooks/useFetcher'; + +export const AnomalyDetection = () => { + const [viewAddEnvironments, setViewAddEnvironments] = useState(false); + + const { refetch, data = [], status } = useFetcher( + (callApmApi) => + callApmApi({ pathname: `/api/apm/settings/anomaly-detection` }), + [], + { preservePreviousData: false } + ); + + const isLoading = + status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING; + const hasFetchFailure = status === FETCH_STATUS.FAILURE; + + return ( + <> + +

+ {i18n.translate('xpack.apm.settings.anomalyDetection.titleText', { + defaultMessage: 'Anomaly detection', + })} +

+ + + + {i18n.translate('xpack.apm.settings.anomalyDetection.descriptionText', { + defaultMessage: + 'The Machine Learning anomaly detection integration enables application health status indicators in the Service map by identifying transaction duration anomalies.', + })} + + + {viewAddEnvironments ? ( + environment)} + onCreateJobSuccess={() => { + refetch(); + setViewAddEnvironments(false); + }} + onCancel={() => { + setViewAddEnvironments(false); + }} + /> + ) : ( + { + setViewAddEnvironments(true); + }} + /> + )} + + ); +}; diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx new file mode 100644 index 00000000000000..30b4805011f03d --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { + EuiPanel, + EuiTitle, + EuiText, + EuiSpacer, + EuiButton, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { ITableColumn, ManagedTable } from '../../../shared/ManagedTable'; +import { LoadingStatePrompt } from '../../../shared/LoadingStatePrompt'; +import { AnomalyDetectionJobByEnv } from '../../../../../typings/anomaly_detection'; +import { MLJobLink } from '../../../shared/Links/MachineLearningLinks/MLJobLink'; +import { MLLink } from '../../../shared/Links/MachineLearningLinks/MLLink'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../../../common/environment_filter_values'; + +const columns: Array> = [ + { + field: 'environment', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.environmentColumnLabel', + { defaultMessage: 'Environment' } + ), + render: (environment: string) => { + if (environment === ENVIRONMENT_NOT_DEFINED) { + return i18n.translate('xpack.apm.filter.environment.notDefinedLabel', { + defaultMessage: 'Not defined', + }); + } + return environment; + }, + }, + { + field: 'job_id', + align: 'right', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.actionColumnLabel', + { defaultMessage: 'Action' } + ), + render: (jobId: string) => ( + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.mlJobLinkText', + { + defaultMessage: 'View job in ML', + } + )} + + ), + }, +]; + +interface Props { + isLoading: boolean; + hasFetchFailure: boolean; + onAddEnvironments: () => void; + anomalyDetectionJobsByEnv: AnomalyDetectionJobByEnv[]; +} +export const JobsList = ({ + isLoading, + hasFetchFailure, + onAddEnvironments, + anomalyDetectionJobsByEnv, +}: Props) => { + return ( + + + + +

+ {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.environments', + { + defaultMessage: 'Environments', + } + )} +

+ + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.addEnvironments', + { + defaultMessage: 'Add environments', + } + )} + + + + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.mlDescriptionText.mlJobsLinkText', + { + defaultMessage: 'Machine Learning', + } + )} + + ), + }} + /> + + + + ) : hasFetchFailure ? ( + + ) : ( + + ) + } + columns={columns} + items={isLoading || hasFetchFailure ? [] : anomalyDetectionJobsByEnv} + /> + + + ); +}; + +function EmptyStatePrompt() { + return ( + <> + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.emptyListText', + { + defaultMessage: 'No anomaly detection jobs.', + } + )} + + ); +} + +function FailureStatePrompt() { + return ( + <> + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.failedFetchText', + { + defaultMessage: 'Unabled to fetch anomaly detection jobs.', + } + )} + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/Settings/index.tsx b/x-pack/plugins/apm/public/components/app/Settings/index.tsx index 578a7db1958d42..6d8571bf577674 100644 --- a/x-pack/plugins/apm/public/components/app/Settings/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Settings/index.tsx @@ -49,12 +49,15 @@ export const Settings: React.FC = (props) => { ), }, { - name: i18n.translate('xpack.apm.settings.indices', { - defaultMessage: 'Indices', - }), - id: '2', - href: getAPMHref('/settings/apm-indices', search), - isSelected: pathname === '/settings/apm-indices', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection', + { + defaultMessage: 'Anomaly detection', + } + ), + id: '4', + href: getAPMHref('/settings/anomaly-detection', search), + isSelected: pathname === '/settings/anomaly-detection', }, { name: i18n.translate('xpack.apm.settings.customizeApp', { @@ -64,6 +67,14 @@ export const Settings: React.FC = (props) => { href: getAPMHref('/settings/customize-ui', search), isSelected: pathname === '/settings/customize-ui', }, + { + name: i18n.translate('xpack.apm.settings.indices', { + defaultMessage: 'Indices', + }), + id: '2', + href: getAPMHref('/settings/apm-indices', search), + isSelected: pathname === '/settings/apm-indices', + }, ], }, ]} diff --git a/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx b/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx index 3dbb1b2faac020..50d46844f0adb7 100644 --- a/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx @@ -33,6 +33,7 @@ interface Props { hidePerPageOptions?: boolean; noItemsMessage?: React.ReactNode; sortItems?: boolean; + pagination?: boolean; } function UnoptimizedManagedTable(props: Props) { @@ -46,6 +47,7 @@ function UnoptimizedManagedTable(props: Props) { hidePerPageOptions = true, noItemsMessage, sortItems = true, + pagination = true, } = props; const { @@ -93,23 +95,26 @@ function UnoptimizedManagedTable(props: Props) { [] ); - const pagination = useMemo(() => { + const paginationProps = useMemo(() => { + if (!pagination) { + return; + } return { hidePerPageOptions, totalItemCount: items.length, pageIndex: page, pageSize, }; - }, [hidePerPageOptions, items, page, pageSize]); + }, [hidePerPageOptions, items, page, pageSize, pagination]); return ( >} // EuiBasicTableColumn is stricter than ITableColumn - pagination={pagination} sorting={sort} onChange={onTableChange} + {...(paginationProps ? { pagination: paginationProps } : {})} /> ); } diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts new file mode 100644 index 00000000000000..406097805775d5 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import uuid from 'uuid/v4'; +import { PromiseReturnType } from '../../../../observability/typings/common'; +import { Setup } from '../helpers/setup_request'; +import { + SERVICE_ENVIRONMENT, + TRANSACTION_DURATION, + PROCESSOR_EVENT, +} from '../../../common/elasticsearch_fieldnames'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; + +const ML_MODULE_ID_APM_TRANSACTION = 'apm_transaction'; +export const ML_GROUP_NAME_APM = 'apm'; + +export type CreateAnomalyDetectionJobsAPIResponse = PromiseReturnType< + typeof createAnomalyDetectionJobs +>; +export async function createAnomalyDetectionJobs( + setup: Setup, + environments: string[], + logger: Logger +) { + const { ml, indices } = setup; + if (!ml) { + logger.warn('Anomaly detection plugin is not available.'); + return []; + } + const mlCapabilities = await ml.mlSystem.mlCapabilities(); + if (!mlCapabilities.mlFeatureEnabledInSpace) { + logger.warn('Anomaly detection feature is not enabled for the space.'); + return []; + } + if (!mlCapabilities.isPlatinumOrTrialLicense) { + logger.warn( + 'Unable to create anomaly detection jobs due to insufficient license.' + ); + return []; + } + logger.info( + `Creating ML anomaly detection jobs for environments: [${environments}].` + ); + + const indexPatternName = indices['apm_oss.transactionIndices']; + const responses = await Promise.all( + environments.map((environment) => + createAnomalyDetectionJob({ ml, environment, indexPatternName }) + ) + ); + const jobResponses = responses.flatMap((response) => response.jobs); + const failedJobs = jobResponses.filter(({ success }) => !success); + + if (failedJobs.length > 0) { + const failedJobIds = failedJobs.map(({ id }) => id).join(', '); + logger.error( + `Failed to create anomaly detection ML jobs for: [${failedJobIds}]:` + ); + failedJobs.forEach(({ error }) => logger.error(JSON.stringify(error))); + throw new Error( + `Failed to create anomaly detection ML jobs for: [${failedJobIds}].` + ); + } + + return jobResponses; +} + +async function createAnomalyDetectionJob({ + ml, + environment, + indexPatternName = 'apm-*-transaction-*', +}: { + ml: Required['ml']; + environment: string; + indexPatternName?: string | undefined; +}) { + const convertedEnvironmentName = convertToMLIdentifier(environment); + const randomToken = uuid().substr(-4); + + return ml.modules.setup({ + moduleId: ML_MODULE_ID_APM_TRANSACTION, + prefix: `${ML_GROUP_NAME_APM}-${convertedEnvironmentName}-${randomToken}-`, + groups: [ML_GROUP_NAME_APM, convertedEnvironmentName], + indexPatternName, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + { exists: { field: TRANSACTION_DURATION } }, + environment === ENVIRONMENT_NOT_DEFINED + ? ENVIRONMENT_NOT_DEFINED_FILTER + : { term: { [SERVICE_ENVIRONMENT]: environment } }, + ], + }, + }, + startDatafeed: true, + jobOverrides: [ + { + custom_settings: { + job_tags: { environment }, + }, + }, + ], + }); +} + +const ENVIRONMENT_NOT_DEFINED_FILTER = { + bool: { + must_not: { + exists: { + field: SERVICE_ENVIRONMENT, + }, + }, + }, +}; + +export function convertToMLIdentifier(value: string) { + return value.replace(/\s+/g, '_').toLowerCase(); +} diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts new file mode 100644 index 00000000000000..252c87e9263db3 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import { PromiseReturnType } from '../../../../observability/typings/common'; +import { Setup } from '../helpers/setup_request'; +import { AnomalyDetectionJobByEnv } from '../../../typings/anomaly_detection'; +import { ML_GROUP_NAME_APM } from './create_anomaly_detection_jobs'; + +export type AnomalyDetectionJobsAPIResponse = PromiseReturnType< + typeof getAnomalyDetectionJobs +>; +export async function getAnomalyDetectionJobs( + setup: Setup, + logger: Logger +): Promise { + const { ml } = setup; + if (!ml) { + return []; + } + try { + const mlCapabilities = await ml.mlSystem.mlCapabilities(); + if ( + !( + mlCapabilities.mlFeatureEnabledInSpace && + mlCapabilities.isPlatinumOrTrialLicense + ) + ) { + logger.warn( + 'Anomaly detection integration is not availble for this user.' + ); + return []; + } + } catch (error) { + logger.warn('Unable to get ML capabilities.'); + logger.error(error); + return []; + } + try { + const { jobs } = await ml.anomalyDetectors.jobs(ML_GROUP_NAME_APM); + return jobs + .map((job) => { + const environment = job.custom_settings?.job_tags?.environment ?? ''; + return { + job_id: job.job_id, + environment, + }; + }) + .filter((job) => job.environment); + } catch (error) { + if (error.statusCode !== 404) { + logger.warn('Unable to get APM ML jobs.'); + logger.error(error); + } + return []; + } +} diff --git a/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap b/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap new file mode 100644 index 00000000000000..b943102b39de82 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap @@ -0,0 +1,85 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`getAllEnvironments fetches all environments 1`] = ` +Object { + "body": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + "missing": undefined, + "size": 100, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "terms": Object { + "processor.event": Array [ + "transaction", + "error", + "metric", + ], + }, + }, + Object { + "term": Object { + "service.name": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], +} +`; + +exports[`getAllEnvironments fetches all environments with includeMissing 1`] = ` +Object { + "body": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + "missing": "ENVIRONMENT_NOT_DEFINED", + "size": 100, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "terms": Object { + "processor.event": Array [ + "transaction", + "error", + "metric", + ], + }, + }, + Object { + "term": Object { + "service.name": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], +} +`; diff --git a/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts new file mode 100644 index 00000000000000..25fc1776947446 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getAllEnvironments } from './get_all_environments'; +import { + SearchParamsMock, + inspectSearchParams, +} from '../../../public/utils/testHelpers'; + +describe('getAllEnvironments', () => { + let mock: SearchParamsMock; + + afterEach(() => { + mock.teardown(); + }); + + it('fetches all environments', async () => { + mock = await inspectSearchParams((setup) => + getAllEnvironments({ + serviceName: 'test', + setup, + }) + ); + + expect(mock.params).toMatchSnapshot(); + }); + + it('fetches all environments with includeMissing', async () => { + mock = await inspectSearchParams((setup) => + getAllEnvironments({ + serviceName: 'test', + setup, + includeMissing: true, + }) + ); + + expect(mock.params).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts similarity index 78% rename from x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts rename to x-pack/plugins/apm/server/lib/environments/get_all_environments.ts index 88a528f12b41c9..9b17033a1f2a5e 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts @@ -4,20 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Setup } from '../../../helpers/setup_request'; +import { Setup } from '../helpers/setup_request'; import { PROCESSOR_EVENT, SERVICE_NAME, SERVICE_ENVIRONMENT, -} from '../../../../../common/elasticsearch_fieldnames'; -import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option'; +} from '../../../common/elasticsearch_fieldnames'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; export async function getAllEnvironments({ serviceName, setup, + includeMissing = false, }: { - serviceName: string | undefined; + serviceName?: string; setup: Setup; + includeMissing?: boolean; }) { const { client, indices } = setup; @@ -49,6 +51,7 @@ export async function getAllEnvironments({ terms: { field: SERVICE_ENVIRONMENT, size: 100, + missing: includeMissing ? ENVIRONMENT_NOT_DEFINED : undefined, }, }, }, @@ -60,5 +63,5 @@ export async function getAllEnvironments({ resp.aggregations?.environments.buckets.map( (bucket) => bucket.key as string ) || []; - return [ALL_OPTION_VALUE, ...environments]; + return environments; } diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts index 14c9378d991928..af073076a812a7 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts @@ -116,6 +116,11 @@ function getMlSetup(context: APMRequestHandlerContext, request: KibanaRequest) { return { mlSystem: ml.mlSystemProvider(mlClient, request), anomalyDetectors: ml.anomalyDetectorsProvider(mlClient, request), + modules: ml.modulesProvider( + mlClient, + request, + context.core.savedObjects.client + ), mlClient, }; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap index db34b4d5d20b5b..24a1840bc0ab87 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap @@ -84,47 +84,6 @@ Object { } `; -exports[`agent configuration queries getAllEnvironments fetches all environments 1`] = ` -Object { - "body": Object { - "aggs": Object { - "environments": Object { - "terms": Object { - "field": "service.environment", - "size": 100, - }, - }, - }, - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "terms": Object { - "processor.event": Array [ - "transaction", - "error", - "metric", - ], - }, - }, - Object { - "term": Object { - "service.name": "foo", - }, - }, - ], - }, - }, - "size": 0, - }, - "index": Array [ - "myIndex", - "myIndex", - "myIndex", - ], -} -`; - exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = ` Object { "body": Object { diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts index d10e06d1df632e..630249052be0b9 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts @@ -4,10 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getAllEnvironments } from './get_all_environments'; +import { getAllEnvironments } from '../../../environments/get_all_environments'; import { Setup } from '../../../helpers/setup_request'; import { PromiseReturnType } from '../../../../../../observability/typings/common'; import { getExistingEnvironmentsForService } from './get_existing_environments_for_service'; +import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option'; export type AgentConfigurationEnvironmentsAPIResponse = PromiseReturnType< typeof getEnvironments @@ -25,7 +26,7 @@ export async function getEnvironments({ getExistingEnvironmentsForService({ serviceName, setup }), ]); - return allEnvironments.map((environment) => { + return [ALL_OPTION_VALUE, ...allEnvironments].map((environment) => { return { name: environment, alreadyConfigured: existingEnvironments.includes(environment), diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts index 515376f8bb18be..5fe9d19ffc8605 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getAllEnvironments } from './get_environments/get_all_environments'; import { getExistingEnvironmentsForService } from './get_environments/get_existing_environments_for_service'; import { getServiceNames } from './get_service_names'; import { listConfigurations } from './list_configurations'; @@ -22,19 +21,6 @@ describe('agent configuration queries', () => { mock.teardown(); }); - describe('getAllEnvironments', () => { - it('fetches all environments', async () => { - mock = await inspectSearchParams((setup) => - getAllEnvironments({ - serviceName: 'foo', - setup, - }) - ); - - expect(mock.params).toMatchSnapshot(); - }); - }); - describe('getExistingEnvironmentsForService', () => { it('fetches unavailable environments', async () => { mock = await inspectSearchParams((setup) => diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index ed1c045616a27c..c314debcd80493 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -81,6 +81,11 @@ import { observabilityDashboardHasDataRoute, observabilityDashboardDataRoute, } from './observability_dashboard'; +import { + anomalyDetectionJobsRoute, + createAnomalyDetectionJobsRoute, + anomalyDetectionEnvironmentsRoute, +} from './settings/anomaly_detection'; const createApmApi = () => { const api = createApi() @@ -170,7 +175,12 @@ const createApmApi = () => { // Observability dashboard .add(observabilityDashboardHasDataRoute) - .add(observabilityDashboardDataRoute); + .add(observabilityDashboardDataRoute) + + // Anomaly detection + .add(anomalyDetectionJobsRoute) + .add(createAnomalyDetectionJobsRoute) + .add(anomalyDetectionEnvironmentsRoute); return api; }; diff --git a/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts b/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts new file mode 100644 index 00000000000000..67eca0da946d0a --- /dev/null +++ b/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { createRoute } from '../create_route'; +import { getAnomalyDetectionJobs } from '../../lib/anomaly_detection/get_anomaly_detection_jobs'; +import { createAnomalyDetectionJobs } from '../../lib/anomaly_detection/create_anomaly_detection_jobs'; +import { setupRequest } from '../../lib/helpers/setup_request'; +import { getAllEnvironments } from '../../lib/environments/get_all_environments'; + +// get ML anomaly detection jobs for each environment +export const anomalyDetectionJobsRoute = createRoute(() => ({ + method: 'GET', + path: '/api/apm/settings/anomaly-detection', + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + return await getAnomalyDetectionJobs(setup, context.logger); + }, +})); + +// create new ML anomaly detection jobs for each given environment +export const createAnomalyDetectionJobsRoute = createRoute(() => ({ + method: 'POST', + path: '/api/apm/settings/anomaly-detection/jobs', + options: { + tags: ['access:apm', 'access:apm_write'], + }, + params: { + body: t.type({ + environments: t.array(t.string), + }), + }, + handler: async ({ context, request }) => { + const { environments } = context.params.body; + const setup = await setupRequest(context, request); + return await createAnomalyDetectionJobs( + setup, + environments, + context.logger + ); + }, +})); + +// get all available environments to create anomaly detection jobs for +export const anomalyDetectionEnvironmentsRoute = createRoute(() => ({ + method: 'GET', + path: '/api/apm/settings/anomaly-detection/environments', + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + return await getAllEnvironments({ setup, includeMissing: true }); + }, +})); diff --git a/x-pack/plugins/apm/typings/anomaly_detection.ts b/x-pack/plugins/apm/typings/anomaly_detection.ts new file mode 100644 index 00000000000000..30dc92c36dea42 --- /dev/null +++ b/x-pack/plugins/apm/typings/anomaly_detection.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface AnomalyDetectionJobByEnv { + environment: string; + job_id: string; +} diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts index 3dbdb8bf3c0024..e2c4f1bae1a108 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts @@ -13,6 +13,9 @@ export type BucketSpan = string; export interface CustomSettings { custom_urls?: UrlConfig[]; created_by?: CREATED_BY_LABEL; + job_tags?: { + [tag: string]: string; + }; } export interface Job {