diff --git a/x-pack/legacy/plugins/ml/common/constants/new_job.ts b/x-pack/legacy/plugins/ml/common/constants/new_job.ts index 004824bef1c9d..ccd108cd2698f 100644 --- a/x-pack/legacy/plugins/ml/common/constants/new_job.ts +++ b/x-pack/legacy/plugins/ml/common/constants/new_job.ts @@ -9,16 +9,24 @@ export enum JOB_TYPE { MULTI_METRIC = 'multi_metric', POPULATION = 'population', ADVANCED = 'advanced', + CATEGORIZATION = 'categorization', } export enum CREATED_BY_LABEL { SINGLE_METRIC = 'single-metric-wizard', MULTI_METRIC = 'multi-metric-wizard', POPULATION = 'population-wizard', + CATEGORIZATION = 'categorization-wizard', } export const DEFAULT_MODEL_MEMORY_LIMIT = '10MB'; export const DEFAULT_BUCKET_SPAN = '15m'; +export const DEFAULT_RARE_BUCKET_SPAN = '1h'; export const DEFAULT_QUERY_DELAY = '60s'; export const SHARED_RESULTS_INDEX_NAME = 'shared'; + +export const NUMBER_OF_CATEGORY_EXAMPLES = 5; +export const CATEGORY_EXAMPLES_MULTIPLIER = 20; +export const CATEGORY_EXAMPLES_WARNING_LIMIT = 0.75; +export const CATEGORY_EXAMPLES_ERROR_LIMIT = 0.2; diff --git a/x-pack/legacy/plugins/ml/common/types/categories.ts b/x-pack/legacy/plugins/ml/common/types/categories.ts new file mode 100644 index 0000000000000..6ccd13ed9a39e --- /dev/null +++ b/x-pack/legacy/plugins/ml/common/types/categories.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type CategoryId = number; + +export interface Category { + job_id: string; + category_id: CategoryId; + terms: string; + regex: string; + max_matching_length: number; + examples: string[]; + grok_pattern: string; +} + +export interface Token { + token: string; + start_offset: number; + end_offset: number; + type: string; + position: number; +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts new file mode 100644 index 0000000000000..cea99eb5ec64c --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEqual } from 'lodash'; +import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { SavedSearchSavedObject } from '../../../../../../common/types/kibana'; +import { JobCreator } from './job_creator'; +import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields'; +import { Job, Datafeed, Detector } from './configs'; +import { createBasicDetector } from './util/default_configs'; +import { + JOB_TYPE, + CREATED_BY_LABEL, + DEFAULT_BUCKET_SPAN, + DEFAULT_RARE_BUCKET_SPAN, +} from '../../../../../../common/constants/new_job'; +import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types'; +import { getRichDetectors } from './util/general'; +import { CategorizationExamplesLoader, CategoryExample } from '../results_loader'; +import { CategorizationAnalyzer, getNewJobDefaults } from '../../../../services/ml_server_info'; + +type CategorizationAnalyzerType = CategorizationAnalyzer | null; + +export class CategorizationJobCreator extends JobCreator { + protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION; + private _createCountDetector: () => void = () => {}; + private _createRareDetector: () => void = () => {}; + private _examplesLoader: CategorizationExamplesLoader; + private _categoryFieldExamples: CategoryExample[] = []; + private _categoryFieldValid: number = 0; + private _detectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE = + ML_JOB_AGGREGATION.COUNT; + private _categorizationAnalyzer: CategorizationAnalyzerType = null; + private _defaultCategorizationAnalyzer: CategorizationAnalyzerType; + + constructor( + indexPattern: IndexPattern, + savedSearch: SavedSearchSavedObject | null, + query: object + ) { + super(indexPattern, savedSearch, query); + this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; + this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query); + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + this._defaultCategorizationAnalyzer = anomalyDetectors.categorization_analyzer || null; + } + + public setDefaultDetectorProperties( + count: Aggregation | null, + rare: Aggregation | null, + eventRate: Field | null + ) { + if (count === null || rare === null || eventRate === null) { + return; + } + + this._createCountDetector = () => { + this._createDetector(count, eventRate); + }; + this._createRareDetector = () => { + this._createDetector(rare, eventRate); + }; + } + + private _createDetector(agg: Aggregation, field: Field) { + const dtr: Detector = createBasicDetector(agg, field); + dtr.by_field_name = mlCategory.id; + this._addDetector(dtr, agg, mlCategory); + } + + public setDetectorType(type: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE) { + this._detectorType = type; + this.removeAllDetectors(); + if (type === ML_JOB_AGGREGATION.COUNT) { + this._createCountDetector(); + this.bucketSpan = DEFAULT_BUCKET_SPAN; + } else { + this._createRareDetector(); + this.bucketSpan = DEFAULT_RARE_BUCKET_SPAN; + this.modelPlot = false; + } + } + + public set categorizationFieldName(fieldName: string | null) { + if (fieldName !== null) { + this._job_config.analysis_config.categorization_field_name = fieldName; + this.setDetectorType(this._detectorType); + this.addInfluencer(mlCategory.id); + } else { + delete this._job_config.analysis_config.categorization_field_name; + this._categoryFieldExamples = []; + this._categoryFieldValid = 0; + } + } + + public get categorizationFieldName(): string | null { + return this._job_config.analysis_config.categorization_field_name || null; + } + + public async loadCategorizationFieldExamples() { + const { valid, examples } = await this._examplesLoader.loadExamples(); + this._categoryFieldExamples = examples; + this._categoryFieldValid = valid; + return { valid, examples }; + } + + public get categoryFieldExamples() { + return this._categoryFieldExamples; + } + + public get categoryFieldValid() { + return this._categoryFieldValid; + } + + public get selectedDetectorType() { + return this._detectorType; + } + + public set categorizationAnalyzer(analyzer: CategorizationAnalyzerType) { + this._categorizationAnalyzer = analyzer; + + if ( + analyzer === null || + isEqual(this._categorizationAnalyzer, this._defaultCategorizationAnalyzer) + ) { + delete this._job_config.analysis_config.categorization_analyzer; + } else { + this._job_config.analysis_config.categorization_analyzer = analyzer; + } + } + + public get categorizationAnalyzer() { + return this._categorizationAnalyzer; + } + + public cloneFromExistingJob(job: Job, datafeed: Datafeed) { + this._overrideConfigs(job, datafeed); + this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; + const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + + const dtr = detectors[0]; + if (detectors.length && dtr.agg !== null && dtr.field !== null) { + this._detectorType = + dtr.agg.id === ML_JOB_AGGREGATION.COUNT + ? ML_JOB_AGGREGATION.COUNT + : ML_JOB_AGGREGATION.RARE; + + const bs = job.analysis_config.bucket_span; + this.setDetectorType(this._detectorType); + // set the bucketspan back to the original value + // as setDetectorType applies a default + this.bucketSpan = bs; + } + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts index 8422223ad91fb..88bacdf49c38a 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts @@ -9,11 +9,13 @@ export { SingleMetricJobCreator } from './single_metric_job_creator'; export { MultiMetricJobCreator } from './multi_metric_job_creator'; export { PopulationJobCreator } from './population_job_creator'; export { AdvancedJobCreator } from './advanced_job_creator'; +export { CategorizationJobCreator } from './categorization_job_creator'; export { JobCreatorType, isSingleMetricJobCreator, isMultiMetricJobCreator, isPopulationJobCreator, isAdvancedJobCreator, + isCategorizationJobCreator, } from './type_guards'; export { jobCreatorFactory } from './job_creator_factory'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts index 868346d0188ea..8655b83a244ad 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts @@ -10,6 +10,7 @@ import { MultiMetricJobCreator } from './multi_metric_job_creator'; import { PopulationJobCreator } from './population_job_creator'; import { AdvancedJobCreator } from './advanced_job_creator'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { CategorizationJobCreator } from './categorization_job_creator'; import { JOB_TYPE } from '../../../../../../common/constants/new_job'; @@ -32,6 +33,9 @@ export const jobCreatorFactory = (jobType: JOB_TYPE) => ( case JOB_TYPE.ADVANCED: jc = AdvancedJobCreator; break; + case JOB_TYPE.CATEGORIZATION: + jc = CategorizationJobCreator; + break; default: jc = SingleMetricJobCreator; break; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts index 9feb0416dd267..25ea80e18eeb3 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts @@ -8,13 +8,15 @@ import { SingleMetricJobCreator } from './single_metric_job_creator'; import { MultiMetricJobCreator } from './multi_metric_job_creator'; import { PopulationJobCreator } from './population_job_creator'; import { AdvancedJobCreator } from './advanced_job_creator'; +import { CategorizationJobCreator } from './categorization_job_creator'; import { JOB_TYPE } from '../../../../../../common/constants/new_job'; export type JobCreatorType = | SingleMetricJobCreator | MultiMetricJobCreator | PopulationJobCreator - | AdvancedJobCreator; + | AdvancedJobCreator + | CategorizationJobCreator; export function isSingleMetricJobCreator( jobCreator: JobCreatorType @@ -37,3 +39,9 @@ export function isPopulationJobCreator( export function isAdvancedJobCreator(jobCreator: JobCreatorType): jobCreator is AdvancedJobCreator { return jobCreator.type === JOB_TYPE.ADVANCED; } + +export function isCategorizationJobCreator( + jobCreator: JobCreatorType +): jobCreator is CategorizationJobCreator { + return jobCreator.type === JOB_TYPE.CATEGORIZATION; +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index 29af08b8826ee..22b727452dd8d 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -19,7 +19,12 @@ import { mlCategory, } from '../../../../../../../common/types/fields'; import { mlJobService } from '../../../../../services/job_service'; -import { JobCreatorType, isMultiMetricJobCreator, isPopulationJobCreator } from '../index'; +import { + JobCreatorType, + isMultiMetricJobCreator, + isPopulationJobCreator, + isCategorizationJobCreator, +} from '../index'; import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job'; const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => { @@ -251,6 +256,8 @@ export function convertToAdvancedJob(jobCreator: JobCreatorType) { jobType = JOB_TYPE.MULTI_METRIC; } else if (isPopulationJobCreator(jobCreator)) { jobType = JOB_TYPE.POPULATION; + } else if (isCategorizationJobCreator(jobCreator)) { + jobType = JOB_TYPE.CATEGORIZATION; } window.location.href = window.location.href.replace(jobType, JOB_TYPE.ADVANCED); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 3c1f767aeaf9c..976e94b377ae8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -12,10 +12,11 @@ import { basicDatafeedValidation, } from '../../../../../../common/util/job_utils'; import { getNewJobLimits } from '../../../../services/ml_server_info'; -import { JobCreator, JobCreatorType } from '../job_creator'; +import { JobCreator, JobCreatorType, isCategorizationJobCreator } from '../job_creator'; import { populateValidationMessages, checkForExistingJobAndGroupIds } from './util'; import { ExistingJobsAndGroups } from '../../../../services/job_service'; import { cardinalityValidator, CardinalityValidatorResult } from './validators'; +import { CATEGORY_EXAMPLES_ERROR_LIMIT } from '../../../../../../common/constants/new_job'; // delay start of validation to allow the user to make changes // e.g. if they are typing in a new value, try not to validate @@ -51,6 +52,10 @@ export interface BasicValidations { scrollSize: Validation; } +export interface AdvancedValidations { + categorizationFieldValid: Validation; +} + export class JobValidator { private _jobCreator: JobCreatorType; private _validationSummary: ValidationSummary; @@ -71,6 +76,9 @@ export class JobValidator { frequency: { valid: true }, scrollSize: { valid: true }, }; + private _advancedValidations: AdvancedValidations = { + categorizationFieldValid: { valid: true }, + }; private _validating: boolean = false; private _basicValidationResult$ = new ReplaySubject(2); @@ -141,6 +149,7 @@ export class JobValidator { this._lastDatafeedConfig = formattedDatafeedConfig; this._validateTimeout = setTimeout(() => { this._runBasicValidation(); + this._runAdvancedValidation(); this._jobCreatorSubject$.next(this._jobCreator); @@ -195,6 +204,13 @@ export class JobValidator { this._basicValidationResult$.next(this._basicValidations); } + private _runAdvancedValidation() { + if (isCategorizationJobCreator(this._jobCreator)) { + this._advancedValidations.categorizationFieldValid.valid = + this._jobCreator.categoryFieldValid > CATEGORY_EXAMPLES_ERROR_LIMIT; + } + } + private _isOverallBasicValid() { return Object.values(this._basicValidations).some(v => v.valid === false) === false; } @@ -246,4 +262,12 @@ export class JobValidator { public get validating(): boolean { return this._validating; } + + public get categorizationField() { + return this._advancedValidations.categorizationFieldValid.valid; + } + + public set categorizationField(valid: boolean) { + this._advancedValidations.categorizationFieldValid.valid = valid; + } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts new file mode 100644 index 0000000000000..16f127ae3d728 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { IndexPatternTitle } from '../../../../../../common/types/kibana'; +import { Token } from '../../../../../../common/types/categories'; +import { CategorizationJobCreator } from '../job_creator'; +import { ml } from '../../../../services/ml_api_service'; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../common/constants/new_job'; + +export interface CategoryExample { + text: string; + tokens: Token[]; +} + +export class CategorizationExamplesLoader { + private _jobCreator: CategorizationJobCreator; + private _indexPatternTitle: IndexPatternTitle = ''; + private _timeFieldName: string = ''; + private _query: object = {}; + + constructor(jobCreator: CategorizationJobCreator, indexPattern: IndexPattern, query: object) { + this._jobCreator = jobCreator; + this._indexPatternTitle = indexPattern.title; + this._query = query; + + if (typeof indexPattern.timeFieldName === 'string') { + this._timeFieldName = indexPattern.timeFieldName; + } + } + + public async loadExamples() { + const analyzer = this._jobCreator.categorizationAnalyzer; + const categorizationFieldName = this._jobCreator.categorizationFieldName; + if (categorizationFieldName === null) { + return { valid: 0, examples: [] }; + } + + const start = Math.floor( + this._jobCreator.start + (this._jobCreator.end - this._jobCreator.start) / 2 + ); + const resp = await ml.jobs.categorizationFieldExamples( + this._indexPatternTitle, + this._query, + NUMBER_OF_CATEGORY_EXAMPLES, + categorizationFieldName, + this._timeFieldName, + start, + 0, + analyzer + ); + return resp; + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts index ef0b05f73fa31..724c62f22e469 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/results_loader/index.ts @@ -5,3 +5,4 @@ */ export { ResultsLoader, Results, ModelItem, Anomaly } from './results_loader'; +export { CategorizationExamplesLoader, CategoryExample } from './categorization_examples_loader'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx index 728229fc3091d..e519b86278ed8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomaly_chart.tsx @@ -7,7 +7,7 @@ import React, { FC } from 'react'; import { Chart, Settings, TooltipType } from '@elastic/charts'; import { ModelItem, Anomaly } from '../../../../common/results_loader'; -import { Anomalies } from './anomalies'; +import { Anomalies } from '../common/anomalies'; import { ModelBounds } from './model_bounds'; import { Line } from './line'; import { Scatter } from './scatter'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomalies.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/anomalies.tsx similarity index 100% rename from x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/anomaly_chart/anomalies.tsx rename to x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/anomalies.tsx diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts index b1852cbb259c1..7dec882429dce 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/common/settings.ts @@ -20,6 +20,7 @@ const themeName = IS_DARK_THEME ? darkTheme : lightTheme; export const LINE_COLOR = themeName.euiColorPrimary; export const MODEL_COLOR = themeName.euiColorPrimary; export const EVENT_RATE_COLOR = themeName.euiColorPrimary; +export const EVENT_RATE_COLOR_WITH_ANOMALIES = themeName.euiColorLightShade; export interface ChartSettings { width: string; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx index 5423e80a82f15..ddbeb3f0f5b04 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/event_rate_chart/event_rate_chart.tsx @@ -8,24 +8,32 @@ import React, { FC } from 'react'; import { BarSeries, Chart, ScaleType, Settings, TooltipType } from '@elastic/charts'; import { Axes } from '../common/axes'; import { LineChartPoint } from '../../../../common/chart_loader'; -import { EVENT_RATE_COLOR } from '../common/settings'; +import { Anomaly } from '../../../../common/results_loader'; +import { EVENT_RATE_COLOR, EVENT_RATE_COLOR_WITH_ANOMALIES } from '../common/settings'; import { LoadingWrapper } from '../loading_wrapper'; +import { Anomalies } from '../common/anomalies'; interface Props { eventRateChartData: LineChartPoint[]; + anomalyData?: Anomaly[]; height: string; width: string; showAxis?: boolean; loading?: boolean; + fadeChart?: boolean; } export const EventRateChart: FC = ({ eventRateChartData, + anomalyData, height, width, showAxis, loading = false, + fadeChart, }) => { + const barColor = fadeChart ? EVENT_RATE_COLOR_WITH_ANOMALIES : EVENT_RATE_COLOR; + return (
= ({ {showAxis === true && } + = ({ xAccessor={'time'} yAccessors={['value']} data={eventRateChartData} - customSeriesColors={[EVENT_RATE_COLOR]} + customSeriesColors={[barColor]} /> diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx index 1087fdc8d0fce..b8600489a4bd9 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/charts/loading_wrapper/loading_wrapper.tsx @@ -9,7 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner } from '@elastic/eui'; interface Props { hasData: boolean; - height: string; + height?: string; loading?: boolean; } @@ -31,7 +31,7 @@ export const LoadingWrapper: FC = ({ hasData, loading = false, height, ch diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx new file mode 100644 index 0000000000000..a44cbf3d0c71a --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/edit_categorization_analyzer_flyout.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useEffect, useState, useContext } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyout, + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButton, + EuiButtonEmpty, + EuiTitle, + EuiFlyoutBody, + EuiSpacer, +} from '@elastic/eui'; +import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor'; +import { isValidJson } from '../../../../../../../../common/util/validation_utils'; +import { JobCreatorContext } from '../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../common/job_creator'; +import { getNewJobDefaults } from '../../../../../../services/ml_server_info'; + +const EDITOR_HEIGHT = '800px'; + +export const EditCategorizationAnalyzerFlyout: FC = () => { + const { jobCreator: jc, jobCreatorUpdate } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + const [showJsonFlyout, setShowJsonFlyout] = useState(false); + const [saveable, setSaveable] = useState(false); + + const [categorizationAnalyzerString, setCategorizationAnalyzerString] = useState( + JSON.stringify(jobCreator.categorizationAnalyzer, null, 2) + ); + + useEffect(() => { + if (showJsonFlyout === true) { + setCategorizationAnalyzerString(JSON.stringify(jobCreator.categorizationAnalyzer, null, 2)); + } + }, [showJsonFlyout]); + + function toggleJsonFlyout() { + setSaveable(false); + setShowJsonFlyout(!showJsonFlyout); + } + + function onJSONChange(json: string) { + setCategorizationAnalyzerString(json); + const valid = isValidJson(json); + setSaveable(valid); + } + + function onSave() { + jobCreator.categorizationAnalyzer = JSON.parse(categorizationAnalyzerString); + jobCreatorUpdate(); + setShowJsonFlyout(false); + } + + function onUseDefault() { + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + const analyzerString = JSON.stringify(anomalyDetectors.categorization_analyzer!, null, 2); + onJSONChange(analyzerString); + } + + return ( + + + + {showJsonFlyout === true && ( + setShowJsonFlyout(false)} hideCloseButton size="m"> + + + + + + + setShowJsonFlyout(false)} + flush="left" + > + + + + + + + + + + + + + + + + + + )} + + ); +}; + +const FlyoutButton: FC<{ onClick(): void }> = ({ onClick }) => { + return ( + + + + ); +}; + +const Contents: FC<{ + title: string; + value: string; + onChange(s: string): void; +}> = ({ title, value, onChange }) => { + return ( + + +
{title}
+ + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts new file mode 100644 index 0000000000000..5bc89a695b6c8 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/edit_categorization_analyzer_flyout/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export { EditCategorizationAnalyzerFlyout } from './edit_categorization_analyzer_flyout'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx index 64f9f450ae08d..a034bdcc2900f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/model_plot/model_plot_switch.tsx @@ -10,16 +10,29 @@ import { EuiSpacer, EuiSwitch } from '@elastic/eui'; import { JobCreatorContext } from '../../../../../job_creator_context'; import { Description } from './description'; import { MMLCallout } from '../mml_callout'; +import { ML_JOB_AGGREGATION } from '../../../../../../../../../../../common/constants/aggregation_types'; +import { isCategorizationJobCreator } from '../../../../../../../common/job_creator'; export const ModelPlotSwitch: FC = () => { - const { jobCreator, jobCreatorUpdate } = useContext(JobCreatorContext); + const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); const [modelPlotEnabled, setModelPlotEnabled] = useState(jobCreator.modelPlot); + const [enabled, setEnabled] = useState(false); useEffect(() => { jobCreator.modelPlot = modelPlotEnabled; jobCreatorUpdate(); }, [modelPlotEnabled]); + useEffect(() => { + const aggs = [ML_JOB_AGGREGATION.RARE]; + // disable model plot switch if the wizard is creating a categorization job + // and a rare detector is being used. + const isRareCategoryJob = + isCategorizationJobCreator(jobCreator) && + jobCreator.aggregations.some(agg => aggs.includes(agg.id)); + setEnabled(isRareCategoryJob === false); + }, [jobCreatorUpdated]); + function toggleModelPlot() { setModelPlotEnabled(!modelPlotEnabled); } @@ -29,6 +42,7 @@ export const ModelPlotSwitch: FC = () => { = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx index dfe9272984b81..216561bac2c62 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span/bucket_span.tsx @@ -14,9 +14,10 @@ import { BucketSpanEstimator } from '../bucket_span_estimator'; interface Props { setIsValid: (proceed: boolean) => void; + hideEstimateButton?: boolean; } -export const BucketSpan: FC = ({ setIsValid }) => { +export const BucketSpan: FC = ({ setIsValid, hideEstimateButton = false }) => { const { jobCreator, jobCreatorUpdate, @@ -56,9 +57,11 @@ export const BucketSpan: FC = ({ setIsValid }) => { disabled={estimating} /> - - - + {hideEstimateButton === false && ( + + + + )} ); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx new file mode 100644 index 0000000000000..96ac1c3f0e325 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/categorization_detector.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiFlexGroup, EuiSpacer, EuiTitle } from '@elastic/eui'; + +import { ML_JOB_AGGREGATION } from '../../../../../../../../../common/constants/aggregation_types'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { CountCard, RareCard } from './detector_cards'; + +export const CategorizationDetector: FC = () => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + const [categorizationDetectorType, setCategorizationDetectorType] = useState( + jobCreator.selectedDetectorType + ); + + useEffect(() => { + if (categorizationDetectorType !== jobCreator.selectedDetectorType) { + jobCreator.setDetectorType(categorizationDetectorType); + jobCreatorUpdate(); + } + }, [categorizationDetectorType]); + + useEffect(() => { + setCategorizationDetectorType(jobCreator.selectedDetectorType); + }, [jobCreatorUpdated]); + + function onCountSelection() { + setCategorizationDetectorType(ML_JOB_AGGREGATION.COUNT); + } + function onRareSelection() { + setCategorizationDetectorType(ML_JOB_AGGREGATION.RARE); + } + + return ( + <> + +

+ +

+ + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx new file mode 100644 index 0000000000000..68d5fc24a96e3 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/detector_cards.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { EuiFlexItem, EuiCard } from '@elastic/eui'; + +interface CardProps { + onClick: () => void; + isSelected: boolean; +} + +export const CountCard: FC = ({ onClick, isSelected }) => ( + + + + + } + selectable={{ onClick, isSelected }} + /> + +); + +export const RareCard: FC = ({ onClick, isSelected }) => ( + + + + + } + selectable={{ onClick, isSelected }} + /> + +); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts new file mode 100644 index 0000000000000..6a13d86d0db9a --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_detector/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export { CategorizationDetector } from './categorization_detector'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx index f9edf79364c97..015300debb156 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx @@ -25,8 +25,10 @@ export const CategorizationField: FC = () => { ); useEffect(() => { - jobCreator.categorizationFieldName = categorizationFieldName; - jobCreatorUpdate(); + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + jobCreator.categorizationFieldName = categorizationFieldName; + jobCreatorUpdate(); + } }, [categorizationFieldName]); useEffect(() => { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx new file mode 100644 index 0000000000000..5017b0c3239e8 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/categorization_view.tsx @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useEffect, useState } from 'react'; +import { EuiHorizontalRule } from '@elastic/eui'; + +import { CategorizationDetectors } from './metric_selection'; +import { CategorizationDetectorsSummary } from './metric_selection_summary'; +import { CategorizationSettings } from './settings'; + +interface Props { + isActive: boolean; + setCanProceed?: (proceed: boolean) => void; +} + +export const CategorizationView: FC = ({ isActive, setCanProceed }) => { + const [categoryFieldValid, setCategoryFieldValid] = useState(false); + const [settingsValid, setSettingsValid] = useState(false); + + useEffect(() => { + if (typeof setCanProceed === 'function') { + setCanProceed(categoryFieldValid && settingsValid); + } + }, [categoryFieldValid, settingsValid]); + + return isActive === false ? ( + + ) : ( + <> + + {categoryFieldValid && ( + <> + + + + )} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx new file mode 100644 index 0000000000000..04934d2dc9a36 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { EuiCallOut, EuiSpacer, EuiCallOutProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { CategorizationAnalyzer } from '../../../../../../../services/ml_server_info'; +import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; +import { + NUMBER_OF_CATEGORY_EXAMPLES, + CATEGORY_EXAMPLES_MULTIPLIER, + CATEGORY_EXAMPLES_ERROR_LIMIT, + CATEGORY_EXAMPLES_WARNING_LIMIT, +} from '../../../../../../../../../common/constants/new_job'; + +type CategorizationAnalyzerType = CategorizationAnalyzer | null; + +interface Props { + examplesValid: number; + categorizationAnalyzer: CategorizationAnalyzerType; +} + +export const ExamplesValidCallout: FC = ({ examplesValid, categorizationAnalyzer }) => { + const percentageText = ; + const analyzerUsed = ; + + let color: EuiCallOutProps['color'] = 'success'; + let title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.valid', + { + defaultMessage: 'Selected category field is valid', + } + ); + + if (examplesValid < CATEGORY_EXAMPLES_ERROR_LIMIT) { + color = 'danger'; + title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.invalid', + { + defaultMessage: 'Selected category field is invalid', + } + ); + } else if (examplesValid < CATEGORY_EXAMPLES_WARNING_LIMIT) { + color = 'warning'; + title = i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldCalloutTitle.possiblyInvalid', + { + defaultMessage: 'Selected category field is possibly invalid', + } + ); + } + + return ( + + {percentageText} + + {analyzerUsed} + + ); +}; + +const PercentageText: FC<{ examplesValid: number }> = ({ examplesValid }) => ( +
+ +
+); + +const AnalyzerUsed: FC<{ categorizationAnalyzer: CategorizationAnalyzerType }> = ({ + categorizationAnalyzer, +}) => { + let analyzer = ''; + if (typeof categorizationAnalyzer === null) { + return null; + } + + if (typeof categorizationAnalyzer === 'string') { + analyzer = categorizationAnalyzer; + } else { + if (categorizationAnalyzer?.tokenizer !== undefined) { + analyzer = categorizationAnalyzer?.tokenizer!; + } else if (categorizationAnalyzer?.analyzer !== undefined) { + analyzer = categorizationAnalyzer?.analyzer!; + } + } + + return ( + <> +
+ +
+
+ +
+ + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx new file mode 100644 index 0000000000000..7f9b2e43b9005 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiBasicTable, EuiText } from '@elastic/eui'; +import { CategoryExample } from '../../../../../common/results_loader'; + +interface Props { + fieldExamples: CategoryExample[] | null; +} + +const TOKEN_HIGHLIGHT_COLOR = '#b0ccf7'; + +export const FieldExamples: FC = ({ fieldExamples }) => { + if (fieldExamples === null || fieldExamples.length === 0) { + return null; + } + + const columns = [ + { + field: 'example', + name: i18n.translate( + 'xpack.ml.newJob.wizard.pickFieldsStep.categorizationFieldExamples.title', + { + defaultMessage: 'Examples', + } + ), + render: (example: any) => ( + + {example} + + ), + }, + ]; + const items = fieldExamples.map((example, i) => { + const txt = []; + let tokenCounter = 0; + let buffer = ''; + let charCount = 0; + while (charCount < example.text.length) { + const token = example.tokens[tokenCounter]; + if (token && charCount === token.start_offset) { + txt.push(buffer); + buffer = ''; + txt.push({token.token}); + charCount += token.end_offset - token.start_offset; + tokenCounter++; + } else { + buffer += example.text[charCount]; + charCount++; + } + } + txt.push(buffer); + return { example: txt }; + }); + return ; +}; + +const Token: FC = ({ children }) => ( + {children} +); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts new file mode 100644 index 0000000000000..f61dfd88a37bb --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { CategorizationView } from './categorization_view'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx new file mode 100644 index 0000000000000..fda0066f9cd37 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; +import { EuiHorizontalRule } from '@elastic/eui'; + +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { CategorizationField } from '../categorization_field'; +import { CategorizationDetector } from '../categorization_detector'; +import { FieldExamples } from './field_examples'; +import { ExamplesValidCallout } from './examples_valid_callout'; +import { CategoryExample } from '../../../../../common/results_loader'; +import { LoadingWrapper } from '../../../charts/loading_wrapper'; + +interface Props { + setIsValid: (na: boolean) => void; +} + +export const CategorizationDetectors: FC = ({ setIsValid }) => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + + const [loadingData, setLoadingData] = useState(false); + const [start, setStart] = useState(jobCreator.start); + const [end, setEnd] = useState(jobCreator.end); + const [categorizationAnalyzerString, setCategorizationAnalyzerString] = useState( + JSON.stringify(jobCreator.categorizationAnalyzer) + ); + const [fieldExamples, setFieldExamples] = useState(null); + const [examplesValid, setExamplesValid] = useState(0); + + const [categorizationFieldName, setCategorizationFieldName] = useState( + jobCreator.categorizationFieldName + ); + + useEffect(() => { + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + jobCreator.categorizationFieldName = categorizationFieldName; + jobCreatorUpdate(); + } + loadFieldExamples(); + }, [categorizationFieldName]); + + useEffect(() => { + let updateExamples = false; + if (jobCreator.start !== start || jobCreator.end !== end) { + setStart(jobCreator.start); + setEnd(jobCreator.end); + updateExamples = true; + } + const tempCategorizationAnalyzerString = JSON.stringify(jobCreator.categorizationAnalyzer); + if (tempCategorizationAnalyzerString !== categorizationAnalyzerString) { + setCategorizationAnalyzerString(tempCategorizationAnalyzerString); + updateExamples = true; + } + + if (updateExamples) { + loadFieldExamples(); + } + if (jobCreator.categorizationFieldName !== categorizationFieldName) { + setCategorizationFieldName(jobCreator.categorizationFieldName); + } + }, [jobCreatorUpdated]); + + async function loadFieldExamples() { + if (categorizationFieldName !== null) { + setLoadingData(true); + const { valid, examples } = await jobCreator.loadCategorizationFieldExamples(); + setFieldExamples(examples); + setExamplesValid(valid); + setLoadingData(false); + } else { + setFieldExamples(null); + setExamplesValid(0); + } + setIsValid(categorizationFieldName !== null); + } + + useEffect(() => { + jobCreatorUpdate(); + }, [examplesValid]); + + return ( + <> + + + + {loadingData === true && ( + +
+ + )} + {fieldExamples !== null && loadingData === false && ( + <> + + + + )} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx new file mode 100644 index 0000000000000..768d8c394fb8f --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { Results, Anomaly } from '../../../../../common/results_loader'; +import { LineChartPoint } from '../../../../../common/chart_loader'; +import { EventRateChart } from '../../../charts/event_rate_chart'; +import { TopCategories } from './top_categories'; + +const DTR_IDX = 0; + +export const CategorizationDetectorsSummary: FC = () => { + const { jobCreator: jc, chartLoader, resultsLoader, chartInterval } = useContext( + JobCreatorContext + ); + const jobCreator = jc as CategorizationJobCreator; + + const [loadingData, setLoadingData] = useState(false); + const [anomalyData, setAnomalyData] = useState([]); + const [eventRateChartData, setEventRateChartData] = useState([]); + const [jobIsRunning, setJobIsRunning] = useState(false); + + function setResultsWrapper(results: Results) { + const anomalies = results.anomalies[DTR_IDX]; + if (anomalies !== undefined) { + setAnomalyData(anomalies); + } + } + + function watchProgress(progress: number) { + setJobIsRunning(progress > 0); + } + + useEffect(() => { + // subscribe to progress and results + const resultsSubscription = resultsLoader.subscribeToResults(setResultsWrapper); + jobCreator.subscribeToProgress(watchProgress); + loadChart(); + return () => { + resultsSubscription.unsubscribe(); + }; + }, []); + + async function loadChart() { + setLoadingData(true); + try { + const resp = await chartLoader.loadEventRateChart( + jobCreator.start, + jobCreator.end, + chartInterval.getInterval().asMilliseconds() + ); + setEventRateChartData(resp); + } catch (error) { + setEventRateChartData([]); + } + setLoadingData(false); + } + + return ( + <> + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx new file mode 100644 index 0000000000000..55db3d495707d --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/settings.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { BucketSpan } from '../bucket_span'; + +interface Props { + setIsValid: (proceed: boolean) => void; +} + +export const CategorizationSettings: FC = ({ setIsValid }) => { + return ( + <> + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx new file mode 100644 index 0000000000000..3bade07250b46 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useState } from 'react'; +import { EuiBasicTable, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { Results } from '../../../../../common/results_loader'; +import { ml } from '../../../../../../../services/ml_api_service'; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../../../../common/constants/new_job'; + +export const TopCategories: FC = () => { + const { jobCreator: jc, resultsLoader } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + + const [tableRow, setTableRow] = useState>([]); + const [totalCategories, setTotalCategories] = useState(0); + + function setResultsWrapper(results: Results) { + loadTopCats(); + } + + async function loadTopCats() { + const results = await ml.jobs.topCategories(jobCreator.jobId, NUMBER_OF_CATEGORY_EXAMPLES); + setTableRow( + results.categories.map(c => ({ + count: c.count, + example: c.category.examples?.length ? c.category.examples[0] : '', + })) + ); + setTotalCategories(results.total); + } + + useEffect(() => { + // subscribe to result updates + const resultsSubscription = resultsLoader.subscribeToResults(setResultsWrapper); + return () => { + resultsSubscription.unsubscribe(); + }; + }, []); + + const columns = [ + // only include counts if model plot is enabled + ...(jobCreator.modelPlot + ? [ + { + field: 'count', + name: 'count', + width: '100px', + render: (count: any) => ( + + {count} + + ), + }, + ] + : []), + { + field: 'example', + name: 'Example', + render: (example: any) => ( + + {example} + + ), + }, + ]; + + return ( + <> + {totalCategories > 0 && ( + <> +
+ +
+ + + )} + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx index 5e800de755f26..b28a9d3da81dc 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/settings.tsx @@ -4,10 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { JobCreatorContext } from '../../../job_creator_context'; import { BucketSpan } from '../bucket_span'; import { SplitFieldSelector } from '../split_field'; import { Influencers } from '../influencers'; @@ -18,19 +17,6 @@ interface Props { } export const MultiMetricSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx index 46cdf44ce0f7d..b9de755e6c946 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/settings.tsx @@ -4,10 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { JobCreatorContext } from '../../../job_creator_context'; import { BucketSpan } from '../bucket_span'; import { Influencers } from '../influencers'; @@ -16,19 +15,6 @@ interface Props { } export const PopulationSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); - return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx index c750235051a6f..f8e7275cf15bb 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/settings.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useContext, useEffect, useState } from 'react'; +import React, { Fragment, FC, useContext } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; @@ -19,18 +19,7 @@ interface Props { } export const SingleMetricSettings: FC = ({ setIsValid }) => { - const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); - const [bucketSpan, setBucketSpan] = useState(jobCreator.bucketSpan); - - useEffect(() => { - jobCreator.bucketSpan = bucketSpan; - jobCreatorUpdate(); - setIsValid(bucketSpan !== ''); - }, [bucketSpan]); - - useEffect(() => { - setBucketSpan(jobCreator.bucketSpan); - }, [jobCreatorUpdated]); + const { jobCreator } = useContext(JobCreatorContext); const convertToMultiMetric = () => { convertToMultiMetricJob(jobCreator); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx index 795dfc30f954a..bfec49678bc34 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/pick_fields.tsx @@ -15,6 +15,7 @@ import { SingleMetricView } from './components/single_metric_view'; import { MultiMetricView } from './components/multi_metric_view'; import { PopulationView } from './components/population_view'; import { AdvancedView } from './components/advanced_view'; +import { CategorizationView } from './components/categorization_view'; import { JsonEditorFlyout, EDITOR_MODE } from '../common/json_editor_flyout'; import { DatafeedPreviewFlyout } from '../common/datafeed_preview_flyout'; @@ -30,7 +31,9 @@ export const PickFieldsStep: FC = ({ setCurrentStep, isCurrentStep }) (jobCreator.type === JOB_TYPE.ADVANCED && jobValidator.modelMemoryLimit.valid)) && jobValidator.bucketSpan.valid && jobValidator.duplicateDetectors.valid && - jobValidator.validating === false; + jobValidator.validating === false && + (jobCreator.type !== JOB_TYPE.CATEGORIZATION || + (jobCreator.type === JOB_TYPE.CATEGORIZATION && jobValidator.categorizationField)); setNextActive(active); }, [jobValidatorUpdated]); @@ -50,6 +53,9 @@ export const PickFieldsStep: FC = ({ setCurrentStep, isCurrentStep }) {jobType === JOB_TYPE.ADVANCED && ( )} + {jobType === JOB_TYPE.CATEGORIZATION && ( + + )} setCurrentStep( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx index f72ff6cf985e5..564228604d71e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/components/detector_chart/detector_chart.tsx @@ -11,6 +11,7 @@ import { SingleMetricView } from '../../../pick_fields_step/components/single_me import { MultiMetricView } from '../../../pick_fields_step/components/multi_metric_view'; import { PopulationView } from '../../../pick_fields_step/components/population_view'; import { AdvancedView } from '../../../pick_fields_step/components/advanced_view'; +import { CategorizationView } from '../../../pick_fields_step/components/categorization_view'; export const DetectorChart: FC = () => { const { jobCreator } = useContext(JobCreatorContext); @@ -21,6 +22,7 @@ export const DetectorChart: FC = () => { {jobCreator.type === JOB_TYPE.MULTI_METRIC && } {jobCreator.type === JOB_TYPE.POPULATION && } {jobCreator.type === JOB_TYPE.ADVANCED && } + {jobCreator.type === JOB_TYPE.CATEGORIZATION && } ); }; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts index 8500279e742b7..51fc226751ae2 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts @@ -32,15 +32,24 @@ function getWizardUrlFromCloningJob(job: CombinedJob) { const created = job?.custom_settings?.created_by; let page = ''; - if (created === CREATED_BY_LABEL.SINGLE_METRIC) { - page = JOB_TYPE.SINGLE_METRIC; - } else if (created === CREATED_BY_LABEL.MULTI_METRIC) { - page = JOB_TYPE.MULTI_METRIC; - } else if (created === CREATED_BY_LABEL.POPULATION) { - page = JOB_TYPE.POPULATION; - } else { - page = JOB_TYPE.ADVANCED; + switch (created) { + case CREATED_BY_LABEL.SINGLE_METRIC: + page = JOB_TYPE.SINGLE_METRIC; + break; + case CREATED_BY_LABEL.MULTI_METRIC: + page = JOB_TYPE.MULTI_METRIC; + break; + case CREATED_BY_LABEL.POPULATION: + page = JOB_TYPE.POPULATION; + break; + case CREATED_BY_LABEL.CATEGORIZATION: + page = JOB_TYPE.CATEGORIZATION; + break; + default: + page = JOB_TYPE.ADVANCED; + break; } + const indexPatternId = getIndexPatternIdFromName(job.datafeed_config.indices[0]); return `jobs/new_job/${page}?index=${indexPatternId}&_g=()`; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx index dbae1948cbe0f..9a44d561c2d94 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx @@ -151,6 +151,22 @@ export const Page: FC = () => { }), id: 'mlJobTypeLinkAdvancedJob', }, + { + href: getUrl('#jobs/new_job/categorization'), + icon: { + type: 'createAdvancedJob', + ariaLabel: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationAriaLabel', { + defaultMessage: 'Categorization job', + }), + }, + title: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationTitle', { + defaultMessage: 'Categorization', + }), + description: i18n.translate('xpack.ml.newJob.wizard.jobType.categorizationDescription', { + defaultMessage: 'Group log messages into categories and detect anomalies within them.', + }), + id: 'mlJobTypeLinkCategorizationJob', + }, ]; return ( diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 2e025c4931818..3a37934e6203a 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -19,8 +19,12 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { Wizard } from './wizard'; import { WIZARD_STEPS } from '../components/step_types'; -import { jobCreatorFactory, isAdvancedJobCreator } from '../../common/job_creator'; import { getJobCreatorTitle } from '../../common/job_creator/util/general'; +import { + jobCreatorFactory, + isAdvancedJobCreator, + isCategorizationJobCreator, +} from '../../common/job_creator'; import { JOB_TYPE, DEFAULT_MODEL_MEMORY_LIMIT, @@ -34,6 +38,9 @@ import { getTimeFilterRange } from '../../../../components/full_time_range_selec import { TimeBuckets } from '../../../../util/time_buckets'; import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_service'; import { expandCombinedJobConfig } from '../../common/job_creator/configs'; +import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; +import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields'; +import { getNewJobDefaults } from '../../../../services/ml_server_info'; const PAGE_WIDTH = 1200; // document.querySelector('.single-metric-job-container').width(); const BAR_TARGET = PAGE_WIDTH > 2000 ? 1000 : PAGE_WIDTH / 2; @@ -66,6 +73,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // cloning a job const clonedJob = mlJobService.cloneJob(mlJobService.tempJobCloningObjects.job); const { job, datafeed } = expandCombinedJobConfig(clonedJob); + initCategorizationSettings(); jobCreator.cloneFromExistingJob(job, datafeed); // if we're not skipping the time range, this is a standard job clone, so wipe the jobId @@ -103,7 +111,11 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // creating a new job jobCreator.bucketSpan = DEFAULT_BUCKET_SPAN; - if (jobCreator.type !== JOB_TYPE.POPULATION && jobCreator.type !== JOB_TYPE.ADVANCED) { + if ( + jobCreator.type !== JOB_TYPE.POPULATION && + jobCreator.type !== JOB_TYPE.ADVANCED && + jobCreator.type !== JOB_TYPE.CATEGORIZATION + ) { // for all other than population or advanced, use 10MB jobCreator.modelMemoryLimit = DEFAULT_MODEL_MEMORY_LIMIT; } @@ -120,6 +132,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { // auto set the time range if creating a new advanced job autoSetTimeRange = isAdvancedJobCreator(jobCreator); + initCategorizationSettings(); } if (autoSetTimeRange && isAdvancedJobCreator(jobCreator)) { @@ -137,6 +150,20 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { } } + function initCategorizationSettings() { + if (isCategorizationJobCreator(jobCreator)) { + // categorization job will always use a count agg, so give it + // to the job creator now + const count = newJobCapsService.getAggById('count'); + const rare = newJobCapsService.getAggById('rare'); + const eventRate = newJobCapsService.getFieldById(EVENT_RATE_FIELD_ID); + jobCreator.setDefaultDetectorProperties(count, rare, eventRate); + + const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults(); + jobCreator.categorizationAnalyzer = anomalyDetectors.categorization_analyzer!; + } + } + const chartInterval = new TimeBuckets(); chartInterval.setBarTarget(BAR_TARGET); chartInterval.setMaxBars(MAX_BARS); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts index d1e81c1c344de..0f19451b23263 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts @@ -4,12 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - IndexPattern, - esQuery, - Query, - esKuery, -} from '../../../../../../../../../src/plugins/data/public'; +import { esQuery, Query, esKuery } from '../../../../../../../../../src/plugins/data/public'; +import { IIndexPattern } from '../../../../../../../../../src/plugins/data/common/index_patterns'; import { KibanaConfigTypeFix } from '../../../contexts/kibana'; import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search'; import { SavedSearchSavedObject } from '../../../../../common/types/kibana'; @@ -19,7 +15,7 @@ import { getQueryFromSavedSearch } from '../../../util/index_utils'; export function createSearchItems( kibanaConfig: KibanaConfigTypeFix, - indexPattern: IndexPattern, + indexPattern: IIndexPattern, savedSearch: SavedSearchSavedObject | null ) { // query is only used by the data visualizer as it needs diff --git a/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx b/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx index ea1baefdce0d1..99c0511cd09ce 100644 --- a/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx +++ b/x-pack/legacy/plugins/ml/public/application/routing/routes/new_job/wizard.tsx @@ -72,6 +72,16 @@ const advancedBreadcrumbs = [ }, ]; +const categorizationBreadcrumbs = [ + ...baseBreadcrumbs, + { + text: i18n.translate('xpack.ml.jobsBreadcrumbs.categorizationLabel', { + defaultMessage: 'Categorization', + }), + href: '', + }, +]; + export const singleMetricRoute: MlRoute = { path: '/jobs/new_job/single_metric', render: (props, config, deps) => ( @@ -104,6 +114,14 @@ export const advancedRoute: MlRoute = { breadcrumbs: advancedBreadcrumbs, }; +export const categorizationRoute: MlRoute = { + path: '/jobs/new_job/categorization', + render: (props, config, deps) => ( + + ), + breadcrumbs: categorizationBreadcrumbs, +}; + const PageWrapper: FC = ({ location, config, jobType, deps }) => { const { index, savedSearchId } = queryString.parse(location.search); const { context, results } = useResolver(index, savedSearchId, config, { diff --git a/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts b/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts index f74260c06567e..3716b9715bb5b 100644 --- a/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts +++ b/x-pack/legacy/plugins/ml/public/application/routing/use_resolver.ts @@ -60,8 +60,6 @@ export const useResolver = ( } } catch (error) { // quietly fail. Let the resolvers handle the redirection if any fail to resolve - // eslint-disable-next-line no-console - console.error('ML page loading resolver', error); } })(); }, []); diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts index 58692754d95d7..2ad2a148f05d1 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts @@ -7,6 +7,7 @@ import { Observable } from 'rxjs'; import { Annotation } from '../../../../common/types/annotations'; import { AggFieldNamePair } from '../../../../common/types/fields'; +import { Category } from '../../../../common/types/categories'; import { ExistingJobsAndGroups } from '../job_service'; import { PrivilegesResponse } from '../../../../common/types/privileges'; import { MlSummaryJobs } from '../../../../common/types/jobs'; @@ -107,7 +108,7 @@ declare interface Ml { checkManageMLPrivileges(): Promise; getJobStats(obj: object): Promise; getDatafeedStats(obj: object): Promise; - esSearch(obj: object): any; + esSearch(obj: object): Promise; esSearch$(obj: object): Observable; getIndices(): Promise; dataRecognizerModuleJobsExist(obj: { moduleId: string }): Promise; @@ -171,6 +172,20 @@ declare interface Ml { start: number, end: number ): Promise<{ progress: number; isRunning: boolean; isJobClosed: boolean }>; + categorizationFieldExamples( + indexPatternTitle: string, + query: object, + size: number, + field: string, + timeField: string | undefined, + start: number, + end: number, + analyzer: any + ): Promise<{ valid: number; examples: any[] }>; + topCategories( + jobId: string, + count: number + ): Promise<{ total: number; categories: Array<{ count?: number; category: Category }> }>; }; estimateBucketSpan(data: BucketSpanEstimatorData): Promise; diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js index 4bec070b2cfdf..05d98dc1a1e64 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/jobs.js @@ -206,4 +206,41 @@ export const jobs = { }, }); }, + + categorizationFieldExamples( + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer + ) { + return http({ + url: `${basePath}/jobs/categorization_field_examples`, + method: 'POST', + data: { + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer, + }, + }); + }, + + topCategories(jobId, count) { + return http({ + url: `${basePath}/jobs/top_categories`, + method: 'POST', + data: { + jobId, + count, + }, + }); + }, }; diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts index 95d670eda8a4f..6bf5a7b0c9743 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_server_info.ts @@ -11,10 +11,18 @@ export interface MlServerDefaults { categorization_examples_limit?: number; model_memory_limit?: string; model_snapshot_retention_days?: number; + categorization_analyzer?: CategorizationAnalyzer; }; datafeeds: { scroll_size?: number }; } +export interface CategorizationAnalyzer { + char_filter?: any[]; + tokenizer?: string; + filter?: any[]; + analyzer?: string; +} + export interface MlServerLimits { max_model_memory_limit?: string; } diff --git a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts index d78c9298c6073..051973d35d8de 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts +++ b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts @@ -15,7 +15,6 @@ import { import { ES_FIELD_TYPES, IIndexPattern, - IndexPattern, IndexPatternsContract, } from '../../../../../../../src/plugins/data/public'; import { ml } from './ml_api_service'; @@ -31,7 +30,7 @@ export function loadNewJobCapabilities( return new Promise(async (resolve, reject) => { if (indexPatternId !== undefined) { // index pattern is being used - const indexPattern: IndexPattern = await indexPatterns.get(indexPatternId); + const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId); await newJobCapsService.initializeFromIndexPattern(indexPattern); resolve(newJobCapsService.newJobCaps); } else if (savedSearchId !== undefined) { diff --git a/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts b/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts index 2b8838c04cf69..2e176b0044314 100644 --- a/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts +++ b/x-pack/legacy/plugins/ml/public/application/util/index_utils.ts @@ -8,7 +8,11 @@ import { toastNotifications } from 'ui/notify'; import { i18n } from '@kbn/i18n'; import chrome from 'ui/chrome'; import { Query } from 'src/plugins/data/public'; -import { IndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; +import { + IndexPattern, + IIndexPattern, + IndexPatternsContract, +} from '../../../../../../../src/plugins/data/public'; import { IndexPatternSavedObject, SavedSearchSavedObject } from '../../../common/types/kibana'; let indexPatternCache: IndexPatternSavedObject[] = []; @@ -71,7 +75,7 @@ export function getIndexPatternIdFromName(name: string) { } export async function getIndexPatternAndSavedSearch(savedSearchId: string) { - const resp: { savedSearch: SavedSearchSavedObject | null; indexPattern: IndexPattern | null } = { + const resp: { savedSearch: SavedSearchSavedObject | null; indexPattern: IIndexPattern | null } = { savedSearch: null, indexPattern: null, }; diff --git a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js index 2ca21efb0bd6a..cf13d329182ba 100644 --- a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js +++ b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js @@ -753,4 +753,29 @@ export const elasticsearchJsPlugin = (Client, config, components) => { ], method: 'GET', }); + + ml.categories = ca({ + urls: [ + { + fmt: '/_xpack/ml/anomaly_detectors/<%=jobId%>/results/categories/<%=categoryId%>', + req: { + jobId: { + type: 'string', + }, + categoryId: { + type: 'string', + }, + }, + }, + { + fmt: '/_xpack/ml/anomaly_detectors/<%=jobId%>/results/categories', + req: { + jobId: { + type: 'string', + }, + }, + }, + ], + method: 'GET', + }); }; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/index.js b/x-pack/legacy/plugins/ml/server/models/job_service/index.js index 78d099dad6606..186bcbae84546 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/index.js +++ b/x-pack/legacy/plugins/ml/server/models/job_service/index.js @@ -8,7 +8,7 @@ import { datafeedsProvider } from './datafeeds'; import { jobsProvider } from './jobs'; import { groupsProvider } from './groups'; import { newJobCapsProvider } from './new_job_caps'; -import { newJobChartsProvider } from './new_job'; +import { newJobChartsProvider, categorizationExamplesProvider } from './new_job'; export function jobServiceProvider(callWithRequest, request) { return { @@ -17,5 +17,6 @@ export function jobServiceProvider(callWithRequest, request) { ...groupsProvider(callWithRequest), ...newJobCapsProvider(callWithRequest, request), ...newJobChartsProvider(callWithRequest, request), + ...categorizationExamplesProvider(callWithRequest, request), }; } diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts new file mode 100644 index 0000000000000..34e871a936088 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/categorization.ts @@ -0,0 +1,294 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; +import { CATEGORY_EXAMPLES_MULTIPLIER } from '../../../../common/constants/new_job'; +import { CategoryId, Category, Token } from '../../../../common/types/categories'; +import { callWithRequestType } from '../../../../common/types/kibana'; + +export function categorizationExamplesProvider(callWithRequest: callWithRequestType) { + async function categorizationExamples( + indexPatternTitle: string, + query: any, + size: number, + categorizationFieldName: string, + timeField: string | undefined, + start: number, + end: number, + analyzer?: any + ) { + if (timeField !== undefined) { + const range = { + range: { + [timeField]: { + gte: start, + format: 'epoch_millis', + }, + }, + }; + + if (query.bool === undefined) { + query.bool = {}; + } + if (query.bool.filter === undefined) { + query.bool.filter = range; + } else { + if (Array.isArray(query.bool.filter)) { + query.bool.filter.push(range); + } else { + query.bool.filter.range = range; + } + } + } + + const results = await callWithRequest('search', { + index: indexPatternTitle, + size, + body: { + _source: categorizationFieldName, + query, + }, + }); + const examples: string[] = results.hits?.hits + ?.map((doc: any) => doc._source[categorizationFieldName]) + .filter((example: string | undefined) => example !== undefined); + + let tokens: Token[] = []; + try { + const { tokens: tempTokens } = await callWithRequest('indices.analyze', { + body: { + ...getAnalyzer(analyzer), + text: examples, + }, + }); + tokens = tempTokens; + } catch (error) { + // fail silently, the tokens could not be loaded + // an empty list of tokens will be returned for each example + } + + const lengths = examples.map(e => e.length); + const sumLengths = lengths.map((s => (a: number) => (s += a))(0)); + + const tokensPerExample: Token[][] = examples.map(e => []); + + tokens.forEach((t, i) => { + for (let g = 0; g < sumLengths.length; g++) { + if (t.start_offset <= sumLengths[g] + g) { + const offset = g > 0 ? sumLengths[g - 1] + g : 0; + tokensPerExample[g].push({ + ...t, + start_offset: t.start_offset - offset, + end_offset: t.end_offset - offset, + }); + break; + } + } + }); + + return examples.map((e, i) => ({ text: e, tokens: tokensPerExample[i] })); + } + + function getAnalyzer(analyzer: any) { + if (typeof analyzer === 'object' && analyzer.tokenizer !== undefined) { + return analyzer; + } else { + return { analyzer: 'standard' }; + } + } + + async function validateCategoryExamples( + indexPatternTitle: string, + query: any, + size: number, + categorizationFieldName: string, + timeField: string | undefined, + start: number, + end: number, + analyzer?: any + ) { + const examples = await categorizationExamples( + indexPatternTitle, + query, + size * CATEGORY_EXAMPLES_MULTIPLIER, + categorizationFieldName, + timeField, + start, + end, + analyzer + ); + + const sortedExamples = examples + .map((e, i) => ({ ...e, origIndex: i })) + .sort((a, b) => b.tokens.length - a.tokens.length); + const validExamples = sortedExamples.filter(e => e.tokens.length > 1); + + return { + valid: sortedExamples.length === 0 ? 0 : validExamples.length / sortedExamples.length, + examples: sortedExamples + .filter( + (e, i) => + i / CATEGORY_EXAMPLES_MULTIPLIER - Math.floor(i / CATEGORY_EXAMPLES_MULTIPLIER) === 0 + ) + .sort((a, b) => a.origIndex - b.origIndex) + .map(e => ({ text: e.text, tokens: e.tokens })), + }; + } + + async function getTotalCategories(jobId: string): Promise<{ total: number }> { + const totalResp = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + { + exists: { + field: 'category_id', + }, + }, + ], + }, + }, + }, + }); + return totalResp?.hits?.total?.value ?? 0; + } + + async function getTopCategoryCounts(jobId: string, numberOfCategories: number) { + const top = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + { + term: { + result_type: 'model_plot', + }, + }, + { + term: { + by_field_name: 'mlcategory', + }, + }, + ], + }, + }, + aggs: { + cat_count: { + terms: { + field: 'by_field_value', + size: numberOfCategories, + }, + }, + }, + }, + }); + + const catCounts: Array<{ + id: CategoryId; + count: number; + }> = top.aggregations?.cat_count?.buckets.map((c: any) => ({ + id: c.key, + count: c.doc_count, + })); + return catCounts || []; + } + + async function getCategories( + jobId: string, + catIds: CategoryId[], + size: number + ): Promise { + const categoryFilter = catIds.length + ? { + terms: { + category_id: catIds, + }, + } + : { + exists: { + field: 'category_id', + }, + }; + const result = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size, + body: { + query: { + bool: { + filter: [ + { + term: { + job_id: jobId, + }, + }, + categoryFilter, + ], + }, + }, + }, + }); + + return result.hits.hits?.map((c: { _source: Category }) => c._source) || []; + } + + async function topCategories(jobId: string, numberOfCategories: number) { + const catCounts = await getTopCategoryCounts(jobId, numberOfCategories); + const categories = await getCategories( + jobId, + catCounts.map(c => c.id), + catCounts.length || numberOfCategories + ); + + const catsById = categories.reduce((p, c) => { + p[c.category_id] = c; + return p; + }, {} as { [id: number]: Category }); + + const total = await getTotalCategories(jobId); + + if (catCounts.length) { + return { + total, + categories: catCounts.map(({ id, count }) => { + return { + count, + category: catsById[id] ?? null, + }; + }), + }; + } else { + return { + total, + categories: categories.map(category => { + return { + category, + }; + }), + }; + } + } + + return { + categorizationExamples, + validateCategoryExamples, + topCategories, + }; +} diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts index a8b6ba494a070..88ae8caa91e4a 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/charts.ts @@ -6,7 +6,7 @@ import { newJobLineChartProvider } from './line_chart'; import { newJobPopulationChartProvider } from './population_chart'; -export type callWithRequestType = (action: string, params: any) => Promise; +import { callWithRequestType } from '../../../../common/types/kibana'; export function newJobChartsProvider(callWithRequest: callWithRequestType) { const { newJobLineChart } = newJobLineChartProvider(callWithRequest); diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts index 758b834ed7b3a..da23efa67d0b5 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/index.ts @@ -5,3 +5,4 @@ */ export { newJobChartsProvider } from './charts'; +export { categorizationExamplesProvider } from './categorization'; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts index 5bb0f39982146..c1a5ad5e38ecc 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/line_chart.ts @@ -6,10 +6,9 @@ import { get } from 'lodash'; import { AggFieldNamePair, EVENT_RATE_FIELD_ID } from '../../../../common/types/fields'; +import { callWithRequestType } from '../../../../common/types/kibana'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; -export type callWithRequestType = (action: string, params: any) => Promise; - type DtrIndex = number; type TimeStamp = number; type Value = number | undefined | null; diff --git a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts index 812a135f6cf08..ee35f13c44ee6 100644 --- a/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts +++ b/x-pack/legacy/plugins/ml/server/models/job_service/new_job/population_chart.ts @@ -6,10 +6,9 @@ import { get } from 'lodash'; import { AggFieldNamePair, EVENT_RATE_FIELD_ID } from '../../../../common/types/fields'; +import { callWithRequestType } from '../../../../common/types/kibana'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; -export type callWithRequestType = (action: string, params: any) => Promise; - const OVER_FIELD_EXAMPLES_COUNT = 40; type DtrIndex = number; diff --git a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js index 4205027eb5ce2..54d447c288151 100644 --- a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js +++ b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.js @@ -181,4 +181,22 @@ export function jobRoutes({ commonRouteConfig, elasticsearchPlugin, route }) { ...commonRouteConfig, }, }); + + route({ + method: 'GET', + path: '/api/ml/anomaly_detectors/{jobId}/results/categories/{categoryId}', + handler(request, reply) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const options = { + jobId: request.params.jobId, + categoryId: request.params.categoryId, + }; + return callWithRequest('ml.categories', options) + .then(resp => reply(resp)) + .catch(resp => reply(wrapError(resp))); + }, + config: { + ...commonRouteConfig, + }, + }); } diff --git a/x-pack/legacy/plugins/ml/server/routes/job_service.js b/x-pack/legacy/plugins/ml/server/routes/job_service.js index 78c57670b7ab3..a83b4fa403f65 100644 --- a/x-pack/legacy/plugins/ml/server/routes/job_service.js +++ b/x-pack/legacy/plugins/ml/server/routes/job_service.js @@ -270,4 +270,50 @@ export function jobServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route ...commonRouteConfig, }, }); + + route({ + method: 'POST', + path: '/api/ml/jobs/categorization_field_examples', + handler(request) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const { validateCategoryExamples } = jobServiceProvider(callWithRequest); + const { + indexPatternTitle, + timeField, + query, + size, + field, + start, + end, + analyzer, + } = request.payload; + return validateCategoryExamples( + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer + ).catch(resp => wrapError(resp)); + }, + config: { + ...commonRouteConfig, + }, + }); + + route({ + method: 'POST', + path: '/api/ml/jobs/top_categories', + handler(request) { + const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); + const { topCategories } = jobServiceProvider(callWithRequest); + const { jobId, count } = request.payload; + return topCategories(jobId, count).catch(resp => wrapError(resp)); + }, + config: { + ...commonRouteConfig, + }, + }); }