- Improve job creation with latest updates for the apm_transaction …

…ML module

- Implements job list in settings by reading from `custom_settings.job_tags['service.environment']`
- Add ML module method `createModuleItem` for job configuration
- Don't allow user to type in duplicate environments

x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx

@@ -89,6 +89,9 @@ export const AddEnvironments = ({
  setSelected(nextSelectedOptions);
  }}
  onCreateOption={(searchValue) => {
+ if (currentEnvironments.includes(searchValue)) {
+ return;
+ }
  const newOption = {
  label: searchValue,
  value: searchValue,

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -6,9 +6,9 @@
 
 import { Logger } from 'kibana/server';
 import uuid from 'uuid/v4';
-// import { Job as AnomalyDetectionJob } from '../../../../ml/server';
 import { PromiseReturnType } from '../../../../observability/typings/common';
 import { Setup } from '../helpers/setup_request';
+import { JobResponse } from '../../../../ml/common/types/modules';
 
 export type CreateAnomalyDetectionJobsAPIResponse = PromiseReturnType<
  typeof createAnomalyDetectionJobs
@@ -29,19 +29,47 @@ export async function createAnomalyDetectionJobs(
  mlCapabilities.isPlatinumOrTrialLicense
  )
  ) {
- logger.warn('Anomaly detection integration is not availble for this user.');
+ logger.warn(
+ 'Anomaly detection integration is not available for this user.'
+ );
  return [];
  }
  logger.info(
- `Creating ML anomaly detection jobs for environments: [${environments}]...`
+ `Creating ML anomaly detection jobs for environments: [${environments}].`
  );
 
- const result = await Promise.all(
- environments.map((environment) => {
- return configureAnomalyDetectionJob({ ml, environment });
- })
+ const dataRecognizerConfigResponses = await Promise.all(
+ environments.map((environment) =>
+ configureAnomalyDetectionJob({ ml, environment })
+ )
+ );
+ const newJobResponses = dataRecognizerConfigResponses.reduce(
+ (acc, response) => {
+ return [...acc, ...response.jobs];
+ },
+ [] as JobResponse[]
  );
- return result;
+
+ const failedJobs = newJobResponses.filter(({ success }) => !success);
+
+ if (failedJobs.length > 0) {
+ const allJobsFailed = failedJobs.length === newJobResponses.length;
+
+ logger.error('Failed to create anomaly detection ML jobs.');
+ failedJobs.forEach(({ error }) => logger.error(JSON.stringify(error)));
+
+ if (allJobsFailed) {
+ throw new Error('Failed to setup anomaly detection ML jobs.');
+ }
+ const failedJobIds = failedJobs.map(({ id }) => id);
+ throw new Error(
+ `Some anomaly detection ML jobs failed to setup: [${failedJobIds.join(
+ ', '
+ )}]`
+ );
+ }
+
+ return newJobResponses;
 }
 
 async function configureAnomalyDetectionJob({
@@ -54,46 +82,31 @@ async function configureAnomalyDetectionJob({
  const convertedEnvironmentName = convertToMLIdentifier(environment);
  const randomToken = uuid().substr(-4);
  const moduleId = 'apm_transaction';
- const prefix = `apm-${convertedEnvironmentName}-${randomToken}-`;
- const groups = ['apm', convertedEnvironmentName];
- const indexPatternName = 'apm-*-transaction-*';
- const query = {
- bool: {
- filter: [
- { term: { 'processor.event': 'transaction' } },
- { exists: { field: 'transaction.duration.us' } },
- { term: { 'service.environment': environment } },
- ],
+
+ return ml.modules.createModuleItem(moduleId, {
+ prefix: `apm-${convertedEnvironmentName}-${randomToken}-`,
+ groups: ['apm', convertedEnvironmentName],
+ indexPatternName: 'apm-*-transaction-*',
+ query: {
+ bool: {
+ filter: [
+ { term: { 'processor.event': 'transaction' } },
+ { exists: { field: 'transaction.duration.us' } },
+ { term: { 'service.environment': environment } },
+ ],
+ },
  },
- };
- const useDedicatedIndex = false;
- const startDatafeed = true;
- const start = undefined;
- const end = undefined;
- const jobOverrides = [
- {
- custom_settings: {
- job_tags: {
- 'service.environment': environment,
+ startDatafeed: true,
+ jobOverrides: [
+ {
+ custom_settings: {
+ job_tags: {
+ 'service.environment': environment,
+ },
  },
  },
- },
- ];
- const datafeedOverrides = undefined;
-
- return ml.modules.setupModuleItems(
- moduleId,
- prefix,
- groups,
- indexPatternName,
- query,
- useDedicatedIndex,
- startDatafeed,
- start,
- end,
- jobOverrides, // Typescript Error: '{ job_tags: { 'service.environment': string; }; }' has no properties in common with type 'CustomSettings'.
- datafeedOverrides
- );
+ ],
+ });
 }
 
 export function convertToMLIdentifier(value: string) {

x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts

@@ -5,7 +5,6 @@
  */
 
 import { Logger } from 'kibana/server';
-import { Job as AnomalyDetectionJob } from '../../../../ml/server';
 import { PromiseReturnType } from '../../../../observability/typings/common';
 import { Setup } from '../helpers/setup_request';
 import { AnomalyDetectionJobByEnv } from '../../../typings/anomaly_detection';
@@ -21,32 +20,48 @@ export async function getAnomalyDetectionJobs(
  if (!ml) {
  return [];
  }
- const mlCapabilities = await ml.mlSystem.mlCapabilities();
- if (
- !(
- mlCapabilities.mlFeatureEnabledInSpace &&
- mlCapabilities.isPlatinumOrTrialLicense
- )
- ) {
- logger.warn('Anomaly detection integration is not availble for this user.');
+ try {
+ const mlCapabilities = await ml.mlSystem.mlCapabilities();
+ if (
+ !(
+ mlCapabilities.mlFeatureEnabledInSpace &&
+ mlCapabilities.isPlatinumOrTrialLicense
+ )
+ ) {
+ logger.warn(
+ 'Anomaly detection integration is not availble for this user.'
+ );
+ return [];
+ }
+ } catch (error) {
+ logger.warn('Unable to get ML capabilities.');
+ logger.error(error);
  return [];
  }
- let mlJobs: AnomalyDetectionJob[] = [];
  try {
- mlJobs = (await ml.anomalyDetectors.jobs('apm')).jobs;
+ const { jobs } = await ml.anomalyDetectors.jobs('apm');
+ return jobs.reduce((acc, anomalyDetectionJob) => {
+ if (
+ anomalyDetectionJob.custom_settings?.job_tags?.['service.environment']
+ ) {
+ return [
+ ...acc,
+ {
+ job_id: anomalyDetectionJob.job_id,
+ 'service.environment':
+ anomalyDetectionJob.custom_settings.job_tags[
+ 'service.environment'
+ ],
+ },
+ ];
+ }
+ return acc;
+ }, [] as AnomalyDetectionJobByEnv[]);
  } catch (error) {
- // if (error.statusCode === 404) {
- // return [];
- // }
+ if (error.statusCode !== 404) {
+ logger.warn('Unable to get APM ML jobs.');
+ logger.error(error);
+ }
+ return [];
  }
- // return mlJobs.map(...)
- const exampleApmJobsByEnv: AnomalyDetectionJobByEnv[] = [
- {
- 'service.environment': 'prod',
- job_id: 'apm-prod-high_mean_response_time',
- },
- { 'service.environment': 'dev', job_id: 'apm-dev-high_mean_response_time' },
- { 'service.environment': 'new', job_id: 'apm-new-high_mean_response_time' },
- ];
- return exampleApmJobsByEnv;
 }

x-pack/plugins/apm/server/lib/helpers/setup_request.ts

@@ -115,7 +115,7 @@ function getMlSetup(context: APMRequestHandlerContext, request: KibanaRequest) {
  const mlClient = ml.mlClient.asScoped(request).callAsCurrentUser;
  return {
  mlSystem: ml.mlSystemProvider(mlClient, request),
- anomalyDetectors: ml.anomalyDetectorsProvider(mlClient),
+ anomalyDetectors: ml.anomalyDetectorsProvider(mlClient, request),
  modules: ml.modulesProvider(
  mlClient,
  request,

x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts

@@ -50,11 +50,6 @@ export const anomalyDetectionEnvironmentsRoute = createRoute(() => ({
  path: '/api/apm/settings/anomaly-detection/environments',
  handler: async ({ context, request }) => {
  const setup = await setupRequest(context, request);
- // return await getAllEnvironments({ setup });
-
- // TODO remove dev test data:
- const environments = await getAllEnvironments({ setup });
- const testEnvironments = ['prod', 'dev', 'test', 'staging'];
- return [...environments, ...testEnvironments];
+ return await getAllEnvironments({ setup });
  },
 }));

x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts

@@ -13,6 +13,7 @@ export type BucketSpan = string;
 export interface CustomSettings {
  custom_urls?: UrlConfig[];
  created_by?: CREATED_BY_LABEL;
+ job_tags?: Record<string, string>;
 }
 
 export interface Job {

x-pack/plugins/ml/server/shared_services/providers/modules.ts

@@ -3,10 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
+import { TypeOf } from '@kbn/config-schema';
 import { LegacyAPICaller, KibanaRequest, SavedObjectsClientContract } from 'kibana/server';
 import { DataRecognizer } from '../../models/data_recognizer';
 import { SharedServicesChecks } from '../shared_services';
+import { setupModuleBodySchema } from '../../routes/schemas/modules';
 
 export interface ModulesProvider {
  modulesProvider(
@@ -18,6 +19,10 @@ export interface ModulesProvider {
  getModule: DataRecognizer['getModule'];
  listModules: DataRecognizer['listModules'];
  setupModuleItems: DataRecognizer['setupModuleItems'];
+ createModuleItem(
+ moduleId: string,
+ setupModuleBody: TypeOf<typeof setupModuleBodySchema>
+ ): ReturnType<DataRecognizer['setupModuleItems']>;
  };
 }
 
@@ -58,6 +63,35 @@ export function getModulesProvider({
 
  return dr.setupModuleItems(...args);
  },
+ async createModuleItem(moduleId, setupModuleBody) {
+ const {
+ prefix,
+ groups,
+ indexPatternName,
+ query,
+ useDedicatedIndex,
+ startDatafeed,
+ start,
+ end,
+ jobOverrides,
+ datafeedOverrides,
+ estimateModelMemory,
+ } = setupModuleBody;
+ return this.setupModuleItems(
+ moduleId,
+ prefix,
+ groups,
+ indexPatternName,
+ query,
+ useDedicatedIndex,
+ startDatafeed,
+ start,
+ end,
+ jobOverrides,
+ datafeedOverrides,
+ estimateModelMemory
+ );
+ },
  };
  },
  };