From 75c49240197ed9a52cab36c99bbda592a858b572 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Mon, 26 Oct 2020 22:56:11 -0500 Subject: [PATCH 01/22] [ML] Add support for script fields in wizard and in viz --- x-pack/plugins/ml/common/util/job_utils.ts | 1 - .../util/model_memory_estimator.ts | 1 + .../services/ml_api_service/index.ts | 9 +++- .../results_service/result_service_rx.ts | 8 +++- .../timeseries_search_service.ts | 3 +- .../calculate_model_memory_limit.ts | 11 +++-- .../models/data_visualizer/data_visualizer.ts | 41 +++++++++++++++---- .../models/fields_service/fields_service.ts | 25 +++++++++-- .../job_validation/validate_cardinality.ts | 12 +++++- .../validate_model_memory_limit.ts | 2 +- .../ml/server/routes/job_validation.ts | 13 +++++- .../routes/schemas/job_validation_schema.ts | 1 + 12 files changed, 105 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 878f5a2c71cb9..75a07b6ba5b54 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -95,7 +95,6 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex // Perform extra check to see if the detector is using a scripted field. const scriptFields = Object.keys(job.datafeed_config.script_fields); isSourceDataChartable = - scriptFields.indexOf(dtr.field_name!) === -1 && scriptFields.indexOf(dtr.partition_field_name!) === -1 && scriptFields.indexOf(dtr.by_field_name!) === -1 && scriptFields.indexOf(dtr.over_field_name!) === -1; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/model_memory_estimator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/model_memory_estimator.ts index 6671aaa83abe0..f23807f156576 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/model_memory_estimator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/model_memory_estimator.ts @@ -134,6 +134,7 @@ export const useModelMemoryEstimator = ( // Update model memory estimation payload on the job creator updates useEffect(() => { modelMemoryEstimator.update({ + datafeedConfig: jobCreator.datafeedConfig, analysisConfig: jobCreator.jobConfig.analysis_config, indexPattern: jobCreator.indexPatternTitle, query: jobCreator.datafeedConfig.query, diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index 0deda455df771..5165efbf1a5ab 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -31,7 +31,11 @@ import { FieldHistogramRequestConfig, FieldRequestConfig, } from '../../datavisualizer/index_based/common'; -import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; +import { + DatafeedOverride, + DataRecognizerConfigResponse, + Module, +} from '../../../../common/types/modules'; import { getHttp } from '../../util/dependency_cache'; export interface MlInfoResponse { @@ -627,6 +631,7 @@ export function mlApiServicesProvider(httpService: HttpService) { }, calculateModelMemoryLimit$({ + datafeedConfig, analysisConfig, indexPattern, query, @@ -634,6 +639,7 @@ export function mlApiServicesProvider(httpService: HttpService) { earliestMs, latestMs, }: { + datafeedConfig: DatafeedOverride; analysisConfig: AnalysisConfig; indexPattern: string; query: any; @@ -642,6 +648,7 @@ export function mlApiServicesProvider(httpService: HttpService) { latestMs: number; }) { const body = JSON.stringify({ + datafeedConfig, analysisConfig, indexPattern, query, diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index e1c322910e237..52843590f4215 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -69,7 +69,8 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { timeFieldName: string, earliestMs: number, latestMs: number, - intervalMs: number + intervalMs: number, + scriptFields: any | undefined ): Observable { // Build the criteria to use in the bool filter part of the request. // Add criteria for the time range, entity fields, @@ -155,6 +156,11 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { field: metricFieldName, }, }; + if (scriptFields !== undefined && scriptFields[metricFieldName] !== undefined) { + metricAgg[metricFunction].script = scriptFields[metricFieldName].script; + } else { + metricAgg[metricFunction].field = metricFieldName; + } if (metricFunction === 'percentiles') { metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts index e43ba8c87083a..ec506893a2962 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts @@ -93,7 +93,8 @@ function getMetricData( chartConfig.timeField, earliestMs, latestMs, - intervalMs + intervalMs, + chartConfig?.datafeedConfig?.script_fields ) .pipe( map((resp) => { diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index 2a2c30601e2ca..ed15c931e82f2 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -10,6 +10,7 @@ import { MLCATEGORY } from '../../../common/constants/field_types'; import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs'; import { fieldsServiceProvider } from '../fields_service'; import { MlInfoResponse } from '../../../common/types/ml_server_info'; +import { DatafeedOverride } from '../../../common/types/modules'; export interface ModelMemoryEstimationResult { /** @@ -45,7 +46,8 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => { query: any, timeFieldName: string, earliestMs: number, - latestMs: number + latestMs: number, + datafeedConfig: DatafeedOverride ): Promise<{ overallCardinality: { [key: string]: number }; maxBucketCardinality: { [key: string]: number }; @@ -100,7 +102,8 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => { query, timeFieldName, earliestMs, - latestMs + latestMs, + datafeedConfig ); } @@ -139,6 +142,7 @@ export function calculateModelMemoryLimitProvider(client: IScopedClusterClient) timeFieldName: string, earliestMs: number, latestMs: number, + datafeedConfig: DatafeedOverride, allowMMLGreaterThanMax = false ): Promise { const { body: info } = await asInternalUser.ml.info(); @@ -151,7 +155,8 @@ export function calculateModelMemoryLimitProvider(client: IScopedClusterClient) query, timeFieldName, earliestMs, - latestMs + latestMs, + datafeedConfig ); const { body } = await asInternalUser.ml.estimateModelMemory({ diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 1f59e990096a4..d6b9b45eac443 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -597,12 +597,14 @@ export class DataVisualizer { samplerShardSize: number, timeFieldName: string, earliestMs?: number, - latestMs?: number + latestMs?: number, + datafeedConfig?: any ) { + const datafeedHasScriptFields = typeof datafeedConfig?.script_fields === 'object'; + const index = indexPatternTitle; const size = 0; const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - // Value count aggregation faster way of checking if field exists than using // filter aggregation with exists query. const aggs: Aggs = {}; @@ -611,9 +613,17 @@ export class DataVisualizer { aggs[`${safeFieldName}_count`] = { filter: { exists: { field } }, }; - aggs[`${safeFieldName}_cardinality`] = { + + let cardinalField = { cardinality: { field }, }; + if (datafeedHasScriptFields && datafeedConfig?.script_fields.hasOwnProperty(field)) { + cardinalField = aggs[`${safeFieldName}_cardinality`] = { + cardinality: { script: datafeedConfig?.script_fields[field].script }, + }; + } + + aggs[`${safeFieldName}_cardinality`] = cardinalField; }); const searchBody = { @@ -661,10 +671,27 @@ export class DataVisualizer { }, }); } else { - stats.aggregatableNotExistsFields.push({ - fieldName: field, - existsInDocs: false, - }); + if (datafeedHasScriptFields && datafeedConfig.script_fields.hasOwnProperty(field)) { + const cardinality = get( + aggregations, + [...aggsPath, `${safeFieldName}_cardinality`, 'value'], + 0 + ); + stats.aggregatableExistsFields.push({ + fieldName: field, + existsInDocs: true, + stats: { + sampleCount, + count, + cardinality, + }, + }); + } else { + stats.aggregatableNotExistsFields.push({ + fieldName: field, + existsInDocs: false, + }); + } } }); diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index 68427e98750eb..d580a4f2a0b03 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -9,6 +9,7 @@ import { IScopedClusterClient } from 'kibana/server'; import { duration } from 'moment'; import { parseInterval } from '../../../common/util/parse_interval'; import { initCardinalityFieldsCache } from './fields_aggs_cache'; +import { DatafeedOverride } from '../../../common/types/modules'; /** * Service for carrying out queries to obtain data @@ -35,7 +36,8 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { */ async function getAggregatableFields( index: string | string[], - fieldNames: string[] + fieldNames: string[], + datafeedConfig: DatafeedOverride ): Promise { const { body } = await asCurrentUser.fieldCaps({ index, @@ -43,6 +45,12 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { }); const aggregatableFields: string[] = []; fieldNames.forEach((fieldName) => { + if ( + typeof datafeedConfig?.script_fields === 'object' && + datafeedConfig.script_fields.hasOwnProperty(fieldName) + ) { + aggregatableFields.push(fieldName); + } const fieldInfo = body.fields[fieldName]; const typeKeys = fieldInfo !== undefined ? Object.keys(fieldInfo) : []; if (typeKeys.length > 0) { @@ -67,10 +75,12 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { query: any, timeFieldName: string, earliestMs: number, - latestMs: number + latestMs: number, + datafeedConfig: DatafeedOverride ): Promise<{ [key: string]: number }> { - const aggregatableFields = await getAggregatableFields(index, fieldNames); + const aggregatableFields = await getAggregatableFields(index, fieldNames, datafeedConfig); + // getAggregatableFields doesn't account for scripted or aggregated fields if (aggregatableFields.length === 0) { return {}; } @@ -113,7 +123,14 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { } const aggs = fieldsToAgg.reduce((obj, field) => { - obj[field] = { cardinality: { field } }; + if ( + typeof datafeedConfig?.script_fields === 'object' && + datafeedConfig.script_fields.hasOwnProperty(field) + ) { + obj[field] = { cardinality: { script: datafeedConfig.script_fields[field].script } }; + } else { + obj[field] = { cardinality: { field } }; + } return obj; }, {} as { [field: string]: { cardinality: { field: string } } }); diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts index c5822b863c83d..10d6a3364c014 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts @@ -66,6 +66,7 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida const relevantDetectors = detectors.filter((detector) => { return typeof detector[fieldName] !== 'undefined'; }); + const datafeedConfig = job.datafeed_config; if (relevantDetectors.length > 0) { try { @@ -83,6 +84,12 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida // parse fieldCaps to return an array of just the fields which are aggregatable if (typeof fieldCaps === 'object' && typeof fieldCaps.fields === 'object') { aggregatableFieldNames = uniqueFieldNames.filter((field) => { + if ( + typeof datafeedConfig?.script_fields === 'object' && + datafeedConfig?.script_fields.hasOwnProperty(field) + ) { + return true; + } if (typeof fieldCaps.fields[field] !== 'undefined') { const fieldType = Object.keys(fieldCaps.fields[field])[0]; return fieldCaps.fields[field][fieldType].aggregatable; @@ -96,7 +103,10 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida job.datafeed_config.query, aggregatableFieldNames, 0, - job.data_description.time_field + job.data_description.time_field, + undefined, + undefined, + datafeedConfig ); uniqueFieldNames.forEach((uniqueFieldName) => { diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts index 9733e17e0f379..37e01c8c167f5 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts @@ -64,7 +64,7 @@ export async function validateModelMemoryLimit( job.data_description.time_field, duration!.start as number, duration!.end as number, - true + job.datafeed_config ); // @ts-expect-error const mmlEstimateBytes: number = numeral(modelMemoryLimit).value(); diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index b52043595327b..afd34031b6302 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -30,7 +30,15 @@ export function jobValidationRoutes({ router, mlLicense }: RouteInitialization, client: IScopedClusterClient, payload: CalculateModelMemoryLimitPayload ) { - const { analysisConfig, indexPattern, query, timeFieldName, earliestMs, latestMs } = payload; + const { + datafeedConfig, + analysisConfig, + indexPattern, + query, + timeFieldName, + earliestMs, + latestMs, + } = payload; return calculateModelMemoryLimitProvider(client)( analysisConfig as AnalysisConfig, @@ -38,7 +46,8 @@ export function jobValidationRoutes({ router, mlLicense }: RouteInitialization, query, timeFieldName, earliestMs, - latestMs + latestMs, + datafeedConfig ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts index ddfb49ce42cb8..f786607e70641 100644 --- a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts @@ -20,6 +20,7 @@ export const estimateBucketSpanSchema = schema.object({ }); export const modelMemoryLimitSchema = schema.object({ + datafeedConfig: datafeedConfigSchema, analysisConfig: analysisConfigSchema, indexPattern: schema.string(), query: schema.any(), From d90f1eca4dc1ded8ce61eef7f0e5710db1e4c431 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Oct 2020 14:26:24 -0500 Subject: [PATCH 02/22] [ML] Fix datafeed preview for aggregations not showing if name of agg is not `bucket` --- .../datafeed_preview_flyout/datafeed_preview.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx index 0dd802855ea67..d421efe369210 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx @@ -61,9 +61,17 @@ export const DatafeedPreview: FC<{ if (combinedJob.datafeed_config && combinedJob.datafeed_config.indices.length) { try { const resp = await mlJobService.searchPreview(combinedJob); - const data = resp.aggregations - ? resp.aggregations.buckets.buckets.slice(0, ML_DATA_PREVIEW_COUNT) - : resp.hits.hits; + let data = resp.hits.hits; + let aggField = ''; + if (resp.aggs) aggField = 'aggs'; + if (resp.aggregations) aggField = 'aggregations'; + + if (aggField !== '') { + if (Object.keys(resp[aggField]).length > 0) { + const accessor = Object.keys(resp[aggField])[0]; + data = resp[aggField][accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT); + } + } setPreviewJsonString(JSON.stringify(data, null, 2)); } catch (error) { From b58070878ccdffb1f144f17a02575d6d989f4350 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Oct 2020 14:29:59 -0500 Subject: [PATCH 03/22] [ML] Seems like it can be just aggregations --- .../datafeed_preview_flyout/datafeed_preview.tsx | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx index d421efe369210..cf98625672019 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx @@ -62,15 +62,10 @@ export const DatafeedPreview: FC<{ try { const resp = await mlJobService.searchPreview(combinedJob); let data = resp.hits.hits; - let aggField = ''; - if (resp.aggs) aggField = 'aggs'; - if (resp.aggregations) aggField = 'aggregations'; - - if (aggField !== '') { - if (Object.keys(resp[aggField]).length > 0) { - const accessor = Object.keys(resp[aggField])[0]; - data = resp[aggField][accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT); - } + // the first item under aggregations can be any name + if (typeof resp.aggregations === 'object' && Object.keys(resp.aggregations).length > 0) { + const accessor = Object.keys(resp.aggregations)[0]; + data = resp.aggregations[accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT); } setPreviewJsonString(JSON.stringify(data, null, 2)); From 77a7e921fba222d68bb84f4ebf1761d8e18845dc Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Oct 2020 16:01:47 -0500 Subject: [PATCH 04/22] [ML] Fix AE plotting for aggregated fields --- .../explorer_charts_container_service.js | 3 ++- .../results_service/result_service_rx.ts | 24 +++++++++++++++++-- .../timeseries_search_service.ts | 2 +- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js index b9634f0eac359..deb456ec53317 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js @@ -125,7 +125,8 @@ export const anomalyDataChange = function ( config.timeField, range.min, range.max, - bucketSpanSeconds * 1000 + bucketSpanSeconds * 1000, + config.datafeedConfig ) .toPromise(); } else { diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index 52843590f4215..14f39a7eb91d4 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -20,6 +20,8 @@ import { JobId } from '../../../../common/types/anomaly_detection_jobs'; import { MlApiServices } from '../ml_api_service'; import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; import { CriteriaField } from './index'; +import type { DatafeedOverride } from '../../../../common/types/modules'; +import type { Aggregation } from '../../../../common/types/anomaly_detection_jobs/datafeed'; interface ResultResponse { success: boolean; @@ -70,8 +72,11 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { earliestMs: number, latestMs: number, intervalMs: number, - scriptFields: any | undefined + dataFeedConfig?: DatafeedOverride ): Observable { + const scriptFields: any | undefined = dataFeedConfig?.script_fields; + const aggFields: Aggregation | undefined = dataFeedConfig?.aggregations; + // Build the criteria to use in the bool filter part of the request. // Add criteria for the time range, entity fields, // plus any additional supplied query. @@ -165,7 +170,22 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { if (metricFunction === 'percentiles') { metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; } - body.aggs.byTime.aggs.metric = metricAgg; + + // when the field is an aggregation field, because the field doesn't actually exist in the indices + // we need to pass all the sub aggs from the original datafeed config + // so that we can access the aggregated field + if (typeof aggFields === 'object' && Object.keys(aggFields).length > 0) { + // first item under aggregations can be any name, not necessarily 'buckets' + const accessor = Object.keys(aggFields)[0]; + const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) }; + if (tempAggs.hasOwnProperty(metricFieldName)) { + tempAggs.metric = aggFields[accessor].aggs[metricFieldName]; + delete tempAggs[metricFieldName]; + } + body.aggs.byTime.aggs = tempAggs; + } else { + body.aggs.byTime.aggs.metric = metricAgg; + } } return mlApiServices.esSearch$({ index, body }).pipe( diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts index ec506893a2962..fff37205fd8bd 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts @@ -94,7 +94,7 @@ function getMetricData( earliestMs, latestMs, intervalMs, - chartConfig?.datafeedConfig?.script_fields + chartConfig?.datafeedConfig ) .pipe( map((resp) => { From ecc77ef07a87dfc357ad2dd341eb7d12acfeac2e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Oct 2020 16:02:22 -0500 Subject: [PATCH 05/22] [ML] Export Aggregation interface --- .../plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts index 47ff618ffa77f..f912638c96a2b 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts @@ -30,7 +30,7 @@ export interface ChunkingConfig { time_span?: string; } -interface Aggregation { +export interface Aggregation { buckets: { date_histogram: { field: string; From 8cd07326aa21c766be252f269d9236fd5184b2e3 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 27 Oct 2020 16:07:57 -0500 Subject: [PATCH 06/22] [ML] Fix plotting issue with AE script fields --- .../application/services/results_service/result_service_rx.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index 14f39a7eb91d4..c297860e4b2ed 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -157,9 +157,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { body.aggs.byTime.aggs = {}; const metricAgg: any = { - [metricFunction]: { - field: metricFieldName, - }, + [metricFunction]: {}, }; if (scriptFields !== undefined && scriptFields[metricFieldName] !== undefined) { metricAgg[metricFunction].script = scriptFields[metricFieldName].script; From a930e2103d310966df2c363a92167715bd542f37 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 28 Oct 2020 11:38:13 -0500 Subject: [PATCH 07/22] [ML] Fix typescript --- .../types/anomaly_detection_jobs/datafeed.ts | 6 ++-- x-pack/plugins/ml/common/types/fields.ts | 13 ++++++++ .../results_service/result_service_rx.ts | 2 +- .../calculate_model_memory_limit.ts | 4 +-- .../models/data_visualizer/data_visualizer.ts | 15 ++++------ .../models/fields_service/fields_service.ts | 30 +++++++++++-------- .../ml/server/routes/job_validation.ts | 3 +- 7 files changed, 46 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts index f912638c96a2b..6905096fdffbd 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts @@ -19,7 +19,9 @@ export interface Datafeed { job_id: JobId; query: object; query_delay?: string; - script_fields?: object; + script_fields?: { + [key: string]: any; + }; scroll_size?: number; delayed_data_check_config?: object; indices_options?: IndicesOptions; @@ -31,7 +33,7 @@ export interface ChunkingConfig { } export interface Aggregation { - buckets: { + [key: string]: { date_histogram: { field: string; fixed_interval: string; diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts index 58eddba83db9d..512d12ca53253 100644 --- a/x-pack/plugins/ml/common/types/fields.ts +++ b/x-pack/plugins/ml/common/types/fields.ts @@ -89,3 +89,16 @@ export const mlCategory: Field = { type: ES_FIELD_TYPES.KEYWORD, aggregatable: false, }; + +export interface FieldAggCardinality { + field: string; + percent?: any; +} + +export interface ScriptAggCardinality { + script: any; +} + +export interface AggCardinality { + cardinality: FieldAggCardinality | ScriptAggCardinality; +} diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index c297860e4b2ed..290a969eb1f0c 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -177,7 +177,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { const accessor = Object.keys(aggFields)[0]; const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) }; if (tempAggs.hasOwnProperty(metricFieldName)) { - tempAggs.metric = aggFields[accessor].aggs[metricFieldName]; + tempAggs.metric = tempAggs[metricFieldName]; delete tempAggs[metricFieldName]; } body.aggs.byTime.aggs = tempAggs; diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index ed15c931e82f2..ed64ee74cecb8 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -47,7 +47,7 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => { timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig: DatafeedOverride + datafeedConfig?: DatafeedOverride ): Promise<{ overallCardinality: { [key: string]: number }; maxBucketCardinality: { [key: string]: number }; @@ -142,7 +142,7 @@ export function calculateModelMemoryLimitProvider(client: IScopedClusterClient) timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig: DatafeedOverride, + datafeedConfig?: DatafeedOverride, allowMMLGreaterThanMax = false ): Promise { const { body: info } = await asInternalUser.ml.info(); diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index d6b9b45eac443..81b4d8cb6b344 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -15,6 +15,7 @@ import { buildSamplerAggregation, getSamplerAggregationsResponsePath, } from '../../lib/query_utils'; +import { AggCardinality } from '../../../common/types/fields'; const SAMPLER_TOP_TERMS_THRESHOLD = 100000; const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; @@ -121,12 +122,6 @@ interface AggHistogram { }; } -interface AggCardinality { - cardinality: { - field: string; - }; -} - interface AggTerms { terms: { field: string; @@ -614,13 +609,15 @@ export class DataVisualizer { filter: { exists: { field } }, }; - let cardinalField = { - cardinality: { field }, - }; + let cardinalField: AggCardinality; if (datafeedHasScriptFields && datafeedConfig?.script_fields.hasOwnProperty(field)) { cardinalField = aggs[`${safeFieldName}_cardinality`] = { cardinality: { script: datafeedConfig?.script_fields[field].script }, }; + } else { + cardinalField = { + cardinality: { field }, + }; } aggs[`${safeFieldName}_cardinality`] = cardinalField; diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index d580a4f2a0b03..9f65f58d1c805 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -10,6 +10,7 @@ import { duration } from 'moment'; import { parseInterval } from '../../../common/util/parse_interval'; import { initCardinalityFieldsCache } from './fields_aggs_cache'; import { DatafeedOverride } from '../../../common/types/modules'; +import { AggCardinality } from '../../../common/types/fields'; /** * Service for carrying out queries to obtain data @@ -37,7 +38,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { async function getAggregatableFields( index: string | string[], fieldNames: string[], - datafeedConfig: DatafeedOverride + datafeedConfig?: DatafeedOverride ): Promise { const { body } = await asCurrentUser.fieldCaps({ index, @@ -76,7 +77,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig: DatafeedOverride + datafeedConfig?: DatafeedOverride ): Promise<{ [key: string]: number }> { const aggregatableFields = await getAggregatableFields(index, fieldNames, datafeedConfig); @@ -122,17 +123,22 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { mustCriteria.push(query); } - const aggs = fieldsToAgg.reduce((obj, field) => { - if ( - typeof datafeedConfig?.script_fields === 'object' && - datafeedConfig.script_fields.hasOwnProperty(field) - ) { - obj[field] = { cardinality: { script: datafeedConfig.script_fields[field].script } }; - } else { - obj[field] = { cardinality: { field } }; + const aggs = fieldsToAgg.reduce( + (obj, field) => { + if ( + typeof datafeedConfig?.script_fields === 'object' && + datafeedConfig.script_fields.hasOwnProperty(field) + ) { + obj[field] = { cardinality: { script: datafeedConfig.script_fields[field].script } }; + } else { + obj[field] = { cardinality: { field } }; + } + return obj; + }, + {} as { + [field: string]: AggCardinality; } - return obj; - }, {} as { [field: string]: { cardinality: { field: string } } }); + ); const body = { query: { diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index afd34031b6302..104c236fbedb0 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -19,6 +19,7 @@ import { import { estimateBucketSpanFactory } from '../models/bucket_span_estimator'; import { calculateModelMemoryLimitProvider } from '../models/calculate_model_memory_limit'; import { validateJob, validateCardinality } from '../models/job_validation'; +import { DatafeedOverride } from '../../common/types/modules'; type CalculateModelMemoryLimitPayload = TypeOf; @@ -47,7 +48,7 @@ export function jobValidationRoutes({ router, mlLicense }: RouteInitialization, timeFieldName, earliestMs, latestMs, - datafeedConfig + datafeedConfig as DatafeedOverride ); } From 0455ba10a12410bd33e671e678064bc46e109862 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 3 Nov 2020 18:27:00 -0600 Subject: [PATCH 08/22] [ML] Clean up validation for nested aggregation field --- .../ml/common/util/validation_utils.ts | 19 +++++++++++++++++++ .../results_service/result_service_rx.ts | 7 +++++-- .../models/data_visualizer/data_visualizer.ts | 15 ++++++++------- .../models/fields_service/fields_service.ts | 8 ++++++++ .../job_validation/validate_cardinality.ts | 10 ++++++++++ 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/ml/common/util/validation_utils.ts b/x-pack/plugins/ml/common/util/validation_utils.ts index ee4be34c6f600..b4f424a053b56 100644 --- a/x-pack/plugins/ml/common/util/validation_utils.ts +++ b/x-pack/plugins/ml/common/util/validation_utils.ts @@ -31,3 +31,22 @@ export function isValidJson(json: string) { return false; } } + +export function findAggField(aggs: Record, fieldName: string): any { + let value; + Object.keys(aggs).some(function (k) { + if (k === fieldName) { + value = aggs[k]; + return true; + } + if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') { + value = findAggField(aggs[k], fieldName); + return value !== undefined; + } + }); + return value; +} + +export function isValidAggregationField(aggs: Record, fieldName: string): boolean { + return findAggField(aggs, fieldName) !== undefined; +} diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index e0c3cd83f5474..f1a02028036b5 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -21,6 +21,7 @@ import { MlApiServices } from '../ml_api_service'; import { CriteriaField } from './index'; import type { DatafeedOverride } from '../../../../common/types/modules'; import type { Aggregation } from '../../../../common/types/anomaly_detection_jobs/datafeed'; +import { findAggField } from '../../../../common/util/validation_utils'; interface ResultResponse { success: boolean; @@ -175,8 +176,10 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { // first item under aggregations can be any name, not necessarily 'buckets' const accessor = Object.keys(aggFields)[0]; const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) }; - if (tempAggs.hasOwnProperty(metricFieldName)) { - tempAggs.metric = tempAggs[metricFieldName]; + const foundValue = findAggField(tempAggs, metricFieldName); + + if (foundValue !== undefined) { + tempAggs.metric = foundValue; delete tempAggs[metricFieldName]; } body.aggs.byTime.aggs = tempAggs; diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 81b4d8cb6b344..16a6fab46a89f 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -16,6 +16,7 @@ import { getSamplerAggregationsResponsePath, } from '../../lib/query_utils'; import { AggCardinality } from '../../../common/types/fields'; +import { DatafeedOverride } from '../../../common/types/modules'; const SAMPLER_TOP_TERMS_THRESHOLD = 100000; const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; @@ -593,16 +594,17 @@ export class DataVisualizer { timeFieldName: string, earliestMs?: number, latestMs?: number, - datafeedConfig?: any + datafeedConfig?: DatafeedOverride ) { - const datafeedHasScriptFields = typeof datafeedConfig?.script_fields === 'object'; - const index = indexPatternTitle; const size = 0; const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); + const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; + // Value count aggregation faster way of checking if field exists than using // filter aggregation with exists query. - const aggs: Aggs = {}; + const aggs: Aggs = datafeedAggConfig !== undefined ? { ...datafeedAggConfig } : {}; + aggregatableFields.forEach((field, i) => { const safeFieldName = getSafeAggregationName(field, i); aggs[`${safeFieldName}_count`] = { @@ -610,7 +612,7 @@ export class DataVisualizer { }; let cardinalField: AggCardinality; - if (datafeedHasScriptFields && datafeedConfig?.script_fields.hasOwnProperty(field)) { + if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { cardinalField = aggs[`${safeFieldName}_cardinality`] = { cardinality: { script: datafeedConfig?.script_fields[field].script }, }; @@ -619,7 +621,6 @@ export class DataVisualizer { cardinality: { field }, }; } - aggs[`${safeFieldName}_cardinality`] = cardinalField; }); @@ -668,7 +669,7 @@ export class DataVisualizer { }, }); } else { - if (datafeedHasScriptFields && datafeedConfig.script_fields.hasOwnProperty(field)) { + if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { const cardinality = get( aggregations, [...aggsPath, `${safeFieldName}_cardinality`, 'value'], diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index 45cd823131eb2..7c846af0b3800 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -11,6 +11,7 @@ import { parseInterval } from '../../../common/util/parse_interval'; import { initCardinalityFieldsCache } from './fields_aggs_cache'; import { DatafeedOverride } from '../../../common/types/modules'; import { AggCardinality } from '../../../common/types/fields'; +import { isValidAggregationField } from '../../../common/util/validation_utils'; /** * Service for carrying out queries to obtain data @@ -45,6 +46,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { fields: fieldNames, }); const aggregatableFields: string[] = []; + const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; fieldNames.forEach((fieldName) => { if ( typeof datafeedConfig?.script_fields === 'object' && @@ -52,6 +54,12 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { ) { aggregatableFields.push(fieldName); } + if ( + datafeedAggConfig !== undefined && + isValidAggregationField(datafeedAggConfig, fieldName) + ) { + aggregatableFields.push(fieldName); + } const fieldInfo = body.fields[fieldName]; const typeKeys = fieldInfo !== undefined ? Object.keys(fieldInfo) : []; if (typeKeys.length > 0) { diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts index 10d6a3364c014..ff87e327df5d5 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts @@ -11,6 +11,7 @@ import { validateJobObject } from './validate_job_object'; import { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; import { Detector } from '../../../common/types/anomaly_detection_jobs'; import { MessageId, JobValidationMessage } from '../../../common/constants/messages'; +import { isValidAggregationField } from '../../../common/util/validation_utils'; function isValidCategorizationConfig(job: CombinedJob, fieldName: string): boolean { return ( @@ -79,6 +80,7 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida index: job.datafeed_config.indices.join(','), fields: uniqueFieldNames, }); + const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; let aggregatableFieldNames: string[] = []; // parse fieldCaps to return an array of just the fields which are aggregatable @@ -90,6 +92,14 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida ) { return true; } + // if datafeed has aggregation fields, check recursively if field exist + if ( + datafeedAggConfig !== undefined && + isValidAggregationField(datafeedAggConfig, fieldName) + ) { + return true; + } + if (typeof fieldCaps.fields[field] !== 'undefined') { const fieldType = Object.keys(fieldCaps.fields[field])[0]; return fieldCaps.fields[field][fieldType].aggregatable; From 4a557f5fb06114f7adf2222c344ae9670dfc925a Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Tue, 3 Nov 2020 18:36:19 -0600 Subject: [PATCH 09/22] [ML] Add validation for missing summary count if datafeed has aggregation --- x-pack/plugins/ml/common/constants/messages.ts | 10 ++++++++++ .../ml/server/models/job_validation/job_validation.ts | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/x-pack/plugins/ml/common/constants/messages.ts b/x-pack/plugins/ml/common/constants/messages.ts index a9e4cdc4a0434..1027ee5bf9a89 100644 --- a/x-pack/plugins/ml/common/constants/messages.ts +++ b/x-pack/plugins/ml/common/constants/messages.ts @@ -442,6 +442,16 @@ export const getMessages = once(() => { url: 'https://www.elastic.co/guide/en/elasticsearch/reference/{{version}}/ml-job-resource.html#ml-job-resource', }, + missing_summary_count_field_name: { + status: VALIDATION_STATUS.ERROR, + text: i18n.translate( + 'xpack.ml.models.jobValidation.messages.missingSummaryCountFieldNameMessage', + { + defaultMessage: + 'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.', + } + ), + }, skipped_extended_tests: { status: VALIDATION_STATUS.WARNING, text: i18n.translate('xpack.ml.models.jobValidation.messages.skippedExtendedTestsMessage', { diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index e3fcc69596dc9..3f21ae442df5f 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -100,6 +100,14 @@ export async function validateJob( ...(await validateModelMemoryLimit(client, mlClient, job, duration)) ); } + + // if datafeed has aggregation, require job config to include a valid summary_doc_field_name + const datafeedAggConfig = job.datafeed_config?.aggregations ?? job.datafeed_config?.aggs; + if (datafeedAggConfig !== undefined) { + if (!job.analysis_config?.summary_count_field_name) { + validationMessages.push({ id: 'missing_summary_count_field_name' }); + } + } } else { validationMessages = basicValidation.messages; validationMessages.push({ id: 'skipped_extended_tests' }); From 2712c6f372867a936e6235307682f412d6d3d975 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 4 Nov 2020 14:43:08 -0600 Subject: [PATCH 10/22] [ML] Fix anomaly search broken because max results is 0 --- .../application/services/results_service/results_service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.js b/x-pack/plugins/ml/public/application/services/results_service/results_service.js index 14a725c2e22b7..867f8d6e0e7d7 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.js +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.js @@ -285,7 +285,7 @@ export function resultsServiceProvider(mlApiServices) { influencerFieldValues: { terms: { field: 'influencer_field_value', - size: maxFieldValues, + size: !!maxFieldValues ? maxFieldValues : ANOMALY_SWIM_LANE_HARD_LIMIT, order: { maxAnomalyScore: 'desc', }, @@ -415,7 +415,7 @@ export function resultsServiceProvider(mlApiServices) { influencerFieldValues: { terms: { field: 'influencer_field_value', - size: maxResults !== undefined ? maxResults : 2, + size: !!maxResults ? maxResults : 2, order: { maxAnomalyScore: 'desc', }, From f76039baafd653753fb85291c33df7c87acf5999 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 4 Nov 2020 14:55:37 -0600 Subject: [PATCH 11/22] [ML] Add missing_summary_count_field_name validation to validate.ts test --- .../test/api_integration/apis/ml/job_validation/validate.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts index e4e6adca9640f..cb663115b958b 100644 --- a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts +++ b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts @@ -303,6 +303,12 @@ export default ({ getService }: FtrProviderContext) => { url: `https://www.elastic.co/guide/en/machine-learning/${pkg.branch}/create-jobs.html#model-memory-limits`, status: 'warning', }, + { + id: 'missing_summary_count_field_name', + status: 'error', + text: + 'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.', + }, ]; expect(body.length).to.eql( From 85e177342b59457c09509b7d837935057f1f7b9c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 4 Nov 2020 15:17:59 -0600 Subject: [PATCH 12/22] [ML] Fix order between datafeedConfig and allowMMLGreaterThanMax --- .../calculate_model_memory_limit.ts | 4 ++-- x-pack/plugins/ml/server/routes/job_validation.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index dd38ce9c8bedb..853019de9ebf8 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -145,8 +145,8 @@ export function calculateModelMemoryLimitProvider( timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig?: DatafeedOverride, - allowMMLGreaterThanMax = false + allowMMLGreaterThanMax = false, + datafeedConfig?: DatafeedOverride ): Promise { const { body: info } = await mlClient.info(); const maxModelMemoryLimit = info.limits.max_model_memory_limit?.toUpperCase(); diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index 6018003e8f176..7ee292dba1325 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -53,6 +53,7 @@ export function jobValidationRoutes( timeFieldName, earliestMs, latestMs, + undefined, datafeedConfig as DatafeedOverride ); } From 79f22ddc31a9c584aa152c5d0139f4f31441a782 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 4 Nov 2020 15:20:33 -0600 Subject: [PATCH 13/22] [ML] Fix jest tests --- x-pack/plugins/ml/common/util/job_utils.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/common/util/job_utils.test.ts b/x-pack/plugins/ml/common/util/job_utils.test.ts index a56ccd5208bab..1ea70c0c19b4e 100644 --- a/x-pack/plugins/ml/common/util/job_utils.test.ts +++ b/x-pack/plugins/ml/common/util/job_utils.test.ts @@ -188,8 +188,8 @@ describe('ML - job utils', () => { expect(isTimeSeriesViewDetector(job, 3)).toBe(false); }); - test('returns false for a detector using a script field as a metric field_name', () => { - expect(isTimeSeriesViewDetector(job, 4)).toBe(false); + test('returns true for a detector using a script field as a metric field_name', () => { + expect(isTimeSeriesViewDetector(job, 4)).toBe(true); }); }); @@ -281,6 +281,7 @@ describe('ML - job utils', () => { expect(isSourceDataChartableForDetector(job, 22)).toBe(true); expect(isSourceDataChartableForDetector(job, 23)).toBe(true); expect(isSourceDataChartableForDetector(job, 24)).toBe(true); + expect(isSourceDataChartableForDetector(job, 37)).toBe(true); }); test('returns false for expected detectors', () => { @@ -296,7 +297,6 @@ describe('ML - job utils', () => { expect(isSourceDataChartableForDetector(job, 34)).toBe(false); expect(isSourceDataChartableForDetector(job, 35)).toBe(false); expect(isSourceDataChartableForDetector(job, 36)).toBe(false); - expect(isSourceDataChartableForDetector(job, 37)).toBe(false); }); }); From 739a423764a336b242cdf5592e7abb3f5e9b46f0 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 4 Nov 2020 15:45:56 -0600 Subject: [PATCH 14/22] [ML] Fix calculateModelMemoryLimitProvider order --- .../server/models/job_validation/validate_model_memory_limit.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts index 6d5b5c77c5506..bdf90986e6a01 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts @@ -65,6 +65,7 @@ export async function validateModelMemoryLimit( job.data_description.time_field, duration!.start as number, duration!.end as number, + undefined, job.datafeed_config ); // @ts-expect-error From d96a119950e710b043eb3c89568be037cc203ed2 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 5 Nov 2020 12:27:22 -0600 Subject: [PATCH 15/22] [ML] Rename cardinalityField, combine if statements, add missing_summary_count_field_name validation to advanced wizard --- .../types/anomaly_detection_jobs/datafeed.ts | 13 ++++++------ x-pack/plugins/ml/common/util/job_utils.ts | 20 +++++++++++++++++++ .../common/job_validator/job_validator.ts | 14 +++++++++++++ .../jobs/new_job/common/job_validator/util.ts | 10 ++++++++++ .../summary_count_field/description.tsx | 9 +++++++-- .../summary_count_field.tsx | 14 +++++++++++-- .../results_service/result_service_rx.ts | 5 ++--- .../models/data_visualizer/data_visualizer.ts | 8 ++++---- .../models/job_validation/job_validation.ts | 6 ++---- 9 files changed, 77 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts index 6905096fdffbd..e5294112dc095 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts @@ -19,9 +19,7 @@ export interface Datafeed { job_id: JobId; query: object; query_delay?: string; - script_fields?: { - [key: string]: any; - }; + script_fields?: Record; scroll_size?: number; delayed_data_check_config?: object; indices_options?: IndicesOptions; @@ -32,16 +30,17 @@ export interface ChunkingConfig { time_span?: string; } -export interface Aggregation { - [key: string]: { +export type Aggregation = Record< + string, + { date_histogram: { field: string; fixed_interval: string; }; aggregations?: { [key: string]: any }; aggs?: { [key: string]: any }; - }; -} + } +>; interface IndicesOptions { expand_wildcards?: 'all' | 'open' | 'closed' | 'hidden' | 'none'; diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 75a07b6ba5b54..d3aebd9106f79 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -559,6 +559,26 @@ export function basicDatafeedValidation(datafeed: Datafeed): ValidationResults { }; } +export function basicJobAndDatafeedValidation(job: Job, datafeed: Datafeed): ValidationResults { + const messages: ValidationResults['messages'] = []; + let valid = true; + + if (datafeed && job) { + const datafeedAggConfig = datafeed.aggregations ?? datafeed?.aggs; + if (datafeedAggConfig !== undefined && !job.analysis_config?.summary_count_field_name) { + valid = false; + messages.push({ id: '`missing_summary_count_field_name`' }); + } + } + + return { + messages, + valid, + contains: (id) => messages.some((m) => id === m.id), + find: (id) => messages.find((m) => id === m.id), + }; +} + export function validateModelMemoryLimit(job: Job, limits: MlServerLimits): ValidationResults { const messages: ValidationResults['messages'] = []; let valid = true; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 635322a6c4469..cfa95608c1b88 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -10,6 +10,7 @@ import { map, startWith, tap } from 'rxjs/operators'; import { basicJobValidation, basicDatafeedValidation, + basicJobAndDatafeedValidation, } from '../../../../../../common/util/job_utils'; import { getNewJobLimits } from '../../../../services/ml_server_info'; import { JobCreator, JobCreatorType, isCategorizationJobCreator } from '../job_creator'; @@ -53,6 +54,7 @@ export interface BasicValidations { scrollSize: Validation; categorizerMissingPerPartition: Validation; categorizerVaryingPerPartitionField: Validation; + summaryCountField: Validation; } export interface AdvancedValidations { @@ -80,6 +82,7 @@ export class JobValidator { scrollSize: { valid: true }, categorizerMissingPerPartition: { valid: true }, categorizerVaryingPerPartitionField: { valid: true }, + summaryCountField: { valid: true }, }; private _advancedValidations: AdvancedValidations = { categorizationFieldValid: { valid: true }, @@ -197,6 +200,14 @@ export class JobValidator { datafeedConfig ); + const basicJobAndDatafeedResults = basicJobAndDatafeedValidation(jobConfig, datafeedConfig); + populateValidationMessages( + basicJobAndDatafeedResults, + this._basicValidations, + jobConfig, + datafeedConfig + ); + // run addition job and group id validation const idResults = checkForExistingJobAndGroupIds( this._jobCreator.jobId, @@ -228,6 +239,9 @@ export class JobValidator { public get bucketSpan(): Validation { return this._basicValidations.bucketSpan; } + public get summaryCountField(): Validation { + return this._basicValidations.summaryCountField; + } public get duplicateDetectors(): Validation { return this._basicValidations.duplicateDetectors; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts index 1ce81bf0dcdf0..239abedc352d8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts @@ -193,6 +193,16 @@ export function populateValidationMessages( basicValidations.frequency.valid = false; basicValidations.frequency.message = invalidTimeIntervalMessage(datafeedConfig.frequency); } + if (validationResults.contains('missing_summary_count_field_name')) { + basicValidations.summaryCountField.valid = false; + basicValidations.summaryCountField.message = i18n.translate( + 'xpack.ml.newJob.wizard.validateJob.summaryCountFieldMissing', + { + defaultMessage: + 'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.', + } + ); + } } export function checkForExistingJobAndGroupIds( diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx index 5109718268ac3..6e0c9124af1c2 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx @@ -8,8 +8,13 @@ import React, { memo, FC } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; +import { Validation } from '../../../../../common/job_validator'; -export const Description: FC = memo(({ children }) => { +interface Props { + validation: Validation; +} + +export const Description: FC = memo(({ children, validation }) => { const title = i18n.translate('xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.title', { defaultMessage: 'Summary count field', }); @@ -23,7 +28,7 @@ export const Description: FC = memo(({ children }) => { /> } > - + <>{children} diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx index af759117b8501..70eaa39f71c69 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx @@ -17,13 +17,23 @@ import { import { Description } from './description'; export const SummaryCountField: FC = () => { - const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const { + jobCreator: jc, + jobCreatorUpdate, + jobCreatorUpdated, + jobValidator, + jobValidatorUpdated, + } = useContext(JobCreatorContext); const jobCreator = jc as MultiMetricJobCreator | PopulationJobCreator | AdvancedJobCreator; const { fields } = newJobCapsService; const [summaryCountFieldName, setSummaryCountFieldName] = useState( jobCreator.summaryCountFieldName ); + const [validation, setValidation] = useState(jobValidator.summaryCountField); + useEffect(() => { + setValidation(jobValidator.summaryCountField); + }, [jobValidatorUpdated]); useEffect(() => { jobCreator.summaryCountFieldName = summaryCountFieldName; @@ -35,7 +45,7 @@ export const SummaryCountField: FC = () => { }, [jobCreatorUpdated]); return ( - + { - const scriptFields: any | undefined = dataFeedConfig?.script_fields; - const aggFields: Aggregation | undefined = dataFeedConfig?.aggregations; + const scriptFields = dataFeedConfig?.script_fields; + const aggFields = dataFeedConfig?.aggs ?? dataFeedConfig?.aggregations; // Build the criteria to use in the bool filter part of the request. // Add criteria for the time range, entity fields, diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 16a6fab46a89f..c6da2a8e92e1b 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -611,17 +611,17 @@ export class DataVisualizer { filter: { exists: { field } }, }; - let cardinalField: AggCardinality; + let cardinalityField: AggCardinality; if (datafeedConfig?.script_fields?.hasOwnProperty(field)) { - cardinalField = aggs[`${safeFieldName}_cardinality`] = { + cardinalityField = aggs[`${safeFieldName}_cardinality`] = { cardinality: { script: datafeedConfig?.script_fields[field].script }, }; } else { - cardinalField = { + cardinalityField = { cardinality: { field }, }; } - aggs[`${safeFieldName}_cardinality`] = cardinalField; + aggs[`${safeFieldName}_cardinality`] = cardinalityField; }); const searchBody = { diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index 3f21ae442df5f..50c6e010fbcd0 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -103,10 +103,8 @@ export async function validateJob( // if datafeed has aggregation, require job config to include a valid summary_doc_field_name const datafeedAggConfig = job.datafeed_config?.aggregations ?? job.datafeed_config?.aggs; - if (datafeedAggConfig !== undefined) { - if (!job.analysis_config?.summary_count_field_name) { - validationMessages.push({ id: 'missing_summary_count_field_name' }); - } + if (datafeedAggConfig !== undefined && !job.analysis_config?.summary_count_field_name) { + validationMessages.push({ id: 'missing_summary_count_field_name' }); } } else { validationMessages = basicValidation.messages; From a3a080935fd51a7f1d76b4079ae153c9517b8f94 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 5 Nov 2020 13:07:52 -0600 Subject: [PATCH 16/22] [ML] Fix character escape & disable next step if misisng summaryCountField --- x-pack/plugins/ml/common/util/job_utils.ts | 2 +- .../jobs/new_job/common/job_validator/job_validator.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index d3aebd9106f79..940d151430360 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -567,7 +567,7 @@ export function basicJobAndDatafeedValidation(job: Job, datafeed: Datafeed): Val const datafeedAggConfig = datafeed.aggregations ?? datafeed?.aggs; if (datafeedAggConfig !== undefined && !job.analysis_config?.summary_count_field_name) { valid = false; - messages.push({ id: '`missing_summary_count_field_name`' }); + messages.push({ id: 'missing_summary_count_field_name' }); } } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index cfa95608c1b88..1c012033e97c8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -311,6 +311,7 @@ export class JobValidator { this.duplicateDetectors.valid && this.categorizerMissingPerPartition.valid && this.categorizerVaryingPerPartitionField.valid && + this.summaryCountField.valid && !this.validating && (this._jobCreator.type !== JOB_TYPE.CATEGORIZATION || (this._jobCreator.type === JOB_TYPE.CATEGORIZATION && this.categorizationField)) From 9769d94bbee31eab5bb5def1fa10a25beb9edc05 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 5 Nov 2020 14:25:56 -0600 Subject: [PATCH 17/22] [ML] Fix allowMMLGreaterThanMax changed to undefined previously --- .../server/models/job_validation/validate_model_memory_limit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts index bdf90986e6a01..f72885cf223fd 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_model_memory_limit.ts @@ -65,7 +65,7 @@ export async function validateModelMemoryLimit( job.data_description.time_field, duration!.start as number, duration!.end as number, - undefined, + true, job.datafeed_config ); // @ts-expect-error From f54c8e8dc5d0419538d1270cb91407de4f45bd42 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 5 Nov 2020 17:07:06 -0600 Subject: [PATCH 18/22] [ML] Update datafeedAggregations check --- .../plugins/ml/common/util/datafeed_utils.ts | 22 +++++++++++++++++++ x-pack/plugins/ml/common/util/job_utils.ts | 6 +++-- .../models/data_visualizer/data_visualizer.ts | 5 +++-- .../models/fields_service/fields_service.ts | 8 ++++--- .../models/job_validation/job_validation.ts | 5 +++-- .../job_validation/validate_cardinality.ts | 7 +++--- 6 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/ml/common/util/datafeed_utils.ts diff --git a/x-pack/plugins/ml/common/util/datafeed_utils.ts b/x-pack/plugins/ml/common/util/datafeed_utils.ts new file mode 100644 index 0000000000000..d86ee50baca19 --- /dev/null +++ b/x-pack/plugins/ml/common/util/datafeed_utils.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Aggregation, Datafeed } from '../types/anomaly_detection_jobs'; + +export const getDatafeedAggregations = ( + datafeedConfig: Partial | undefined +): Aggregation | undefined => { + if (datafeedConfig?.aggregations !== undefined) return datafeedConfig.aggregations; + if (datafeedConfig?.aggs !== undefined) return datafeedConfig.aggs; + return undefined; +}; + +export const getAggregationBucketsName = (aggregations: any): string | undefined => { + if (typeof aggregations === 'object') { + const keys = Object.keys(aggregations); + return keys.length > 0 ? keys[0] : undefined; + } +}; diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 940d151430360..1fa1221c075be 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -21,6 +21,7 @@ import { MlServerLimits } from '../types/ml_server_info'; import { JobValidationMessage, JobValidationMessageId } from '../constants/messages'; import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '../constants/aggregation_types'; import { MLCATEGORY } from '../constants/field_types'; +import { getDatafeedAggregations } from './datafeed_utils'; export interface ValidationResults { valid: boolean; @@ -564,8 +565,9 @@ export function basicJobAndDatafeedValidation(job: Job, datafeed: Datafeed): Val let valid = true; if (datafeed && job) { - const datafeedAggConfig = datafeed.aggregations ?? datafeed?.aggs; - if (datafeedAggConfig !== undefined && !job.analysis_config?.summary_count_field_name) { + const datafeedAggregations = getDatafeedAggregations(datafeed); + + if (datafeedAggregations !== undefined && !job.analysis_config?.summary_count_field_name) { valid = false; messages.push({ id: 'missing_summary_count_field_name' }); } diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index c6da2a8e92e1b..dbd780bc31975 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -17,6 +17,7 @@ import { } from '../../lib/query_utils'; import { AggCardinality } from '../../../common/types/fields'; import { DatafeedOverride } from '../../../common/types/modules'; +import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; const SAMPLER_TOP_TERMS_THRESHOLD = 100000; const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; @@ -599,11 +600,11 @@ export class DataVisualizer { const index = indexPatternTitle; const size = 0; const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query); - const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; + const datafeedAggregations = getDatafeedAggregations(datafeedConfig); // Value count aggregation faster way of checking if field exists than using // filter aggregation with exists query. - const aggs: Aggs = datafeedAggConfig !== undefined ? { ...datafeedAggConfig } : {}; + const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {}; aggregatableFields.forEach((field, i) => { const safeFieldName = getSafeAggregationName(field, i); diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index 7c846af0b3800..7455c4634deda 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -12,6 +12,7 @@ import { initCardinalityFieldsCache } from './fields_aggs_cache'; import { DatafeedOverride } from '../../../common/types/modules'; import { AggCardinality } from '../../../common/types/fields'; import { isValidAggregationField } from '../../../common/util/validation_utils'; +import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; /** * Service for carrying out queries to obtain data @@ -46,7 +47,8 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { fields: fieldNames, }); const aggregatableFields: string[] = []; - const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; + const datafeedAggregations = getDatafeedAggregations(datafeedConfig); + fieldNames.forEach((fieldName) => { if ( typeof datafeedConfig?.script_fields === 'object' && @@ -55,8 +57,8 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { aggregatableFields.push(fieldName); } if ( - datafeedAggConfig !== undefined && - isValidAggregationField(datafeedAggConfig, fieldName) + datafeedAggregations !== undefined && + isValidAggregationField(datafeedAggregations, fieldName) ) { aggregatableFields.push(fieldName); } diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index 50c6e010fbcd0..3526f9cebb89b 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -27,6 +27,7 @@ import { validateTimeRange, isValidTimeField } from './validate_time_range'; import { validateJobSchema } from '../../routes/schemas/job_validation_schema'; import { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; import type { MlClient } from '../../lib/ml_client'; +import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; export type ValidateJobPayload = TypeOf; @@ -102,8 +103,8 @@ export async function validateJob( } // if datafeed has aggregation, require job config to include a valid summary_doc_field_name - const datafeedAggConfig = job.datafeed_config?.aggregations ?? job.datafeed_config?.aggs; - if (datafeedAggConfig !== undefined && !job.analysis_config?.summary_count_field_name) { + const datafeedAggregations = getDatafeedAggregations(job.datafeed_config); + if (datafeedAggregations !== undefined && !job.analysis_config?.summary_count_field_name) { validationMessages.push({ id: 'missing_summary_count_field_name' }); } } else { diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts index ff87e327df5d5..c4473d7172ddc 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts @@ -12,6 +12,7 @@ import { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; import { Detector } from '../../../common/types/anomaly_detection_jobs'; import { MessageId, JobValidationMessage } from '../../../common/constants/messages'; import { isValidAggregationField } from '../../../common/util/validation_utils'; +import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; function isValidCategorizationConfig(job: CombinedJob, fieldName: string): boolean { return ( @@ -80,7 +81,7 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida index: job.datafeed_config.indices.join(','), fields: uniqueFieldNames, }); - const datafeedAggConfig = datafeedConfig?.aggregations ?? datafeedConfig?.aggs; + const datafeedAggregations = getDatafeedAggregations(datafeedConfig); let aggregatableFieldNames: string[] = []; // parse fieldCaps to return an array of just the fields which are aggregatable @@ -94,8 +95,8 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida } // if datafeed has aggregation fields, check recursively if field exist if ( - datafeedAggConfig !== undefined && - isValidAggregationField(datafeedAggConfig, fieldName) + datafeedAggregations !== undefined && + isValidAggregationField(datafeedAggregations, fieldName) ) { return true; } From 73555598dd6f29ff6b11152e4a07791cc23fb218 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 5 Nov 2020 17:50:21 -0600 Subject: [PATCH 19/22] [ML] Change DatafeedOverride to Datafeed --- .../application/services/ml_api_service/index.ts | 8 ++------ .../services/results_service/result_service_rx.ts | 10 +++++----- .../calculate_model_memory_limit.ts | 7 +++---- .../server/models/data_visualizer/data_visualizer.ts | 4 ++-- .../ml/server/models/fields_service/fields_service.ts | 6 +++--- x-pack/plugins/ml/server/routes/job_validation.ts | 5 ++--- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index d5aeacf65a174..b67b5015dbd6c 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -32,11 +32,7 @@ import { FieldHistogramRequestConfig, FieldRequestConfig, } from '../../datavisualizer/index_based/common'; -import { - DatafeedOverride, - DataRecognizerConfigResponse, - Module, -} from '../../../../common/types/modules'; +import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; import { getHttp } from '../../util/dependency_cache'; export interface MlInfoResponse { @@ -640,7 +636,7 @@ export function mlApiServicesProvider(httpService: HttpService) { earliestMs, latestMs, }: { - datafeedConfig: DatafeedOverride; + datafeedConfig?: Datafeed; analysisConfig: AnalysisConfig; indexPattern: string; query: any; diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index 9e51d87300856..fb3c068139e51 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -16,11 +16,11 @@ import { map } from 'rxjs/operators'; import { each, get } from 'lodash'; import { Dictionary } from '../../../../common/types/common'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; -import { JobId } from '../../../../common/types/anomaly_detection_jobs'; +import { Datafeed, JobId } from '../../../../common/types/anomaly_detection_jobs'; import { MlApiServices } from '../ml_api_service'; import { CriteriaField } from './index'; -import type { DatafeedOverride } from '../../../../common/types/modules'; import { findAggField } from '../../../../common/util/validation_utils'; +import { getDatafeedAggregations } from '../../../../common/util/datafeed_utils'; interface ResultResponse { success: boolean; @@ -71,10 +71,10 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { earliestMs: number, latestMs: number, intervalMs: number, - dataFeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ): Observable { - const scriptFields = dataFeedConfig?.script_fields; - const aggFields = dataFeedConfig?.aggs ?? dataFeedConfig?.aggregations; + const scriptFields = datafeedConfig?.script_fields; + const aggFields = getDatafeedAggregations(datafeedConfig); // Build the criteria to use in the bool filter part of the request. // Add criteria for the time range, entity fields, diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index 853019de9ebf8..865f305f2ff9f 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -7,10 +7,9 @@ import numeral from '@elastic/numeral'; import { IScopedClusterClient } from 'kibana/server'; import { MLCATEGORY } from '../../../common/constants/field_types'; -import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs'; +import { AnalysisConfig, Datafeed } from '../../../common/types/anomaly_detection_jobs'; import { fieldsServiceProvider } from '../fields_service'; import { MlInfoResponse } from '../../../common/types/ml_server_info'; -import { DatafeedOverride } from '../../../common/types/modules'; import type { MlClient } from '../../lib/ml_client'; export interface ModelMemoryEstimationResult { @@ -48,7 +47,7 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => { timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ): Promise<{ overallCardinality: { [key: string]: number }; maxBucketCardinality: { [key: string]: number }; @@ -146,7 +145,7 @@ export function calculateModelMemoryLimitProvider( earliestMs: number, latestMs: number, allowMMLGreaterThanMax = false, - datafeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ): Promise { const { body: info } = await mlClient.info(); const maxModelMemoryLimit = info.limits.max_model_memory_limit?.toUpperCase(); diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index dbd780bc31975..0142e44276eee 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -16,8 +16,8 @@ import { getSamplerAggregationsResponsePath, } from '../../lib/query_utils'; import { AggCardinality } from '../../../common/types/fields'; -import { DatafeedOverride } from '../../../common/types/modules'; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; +import { Datafeed } from '../../../common/types/anomaly_detection_jobs'; const SAMPLER_TOP_TERMS_THRESHOLD = 100000; const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; @@ -595,7 +595,7 @@ export class DataVisualizer { timeFieldName: string, earliestMs?: number, latestMs?: number, - datafeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ) { const index = indexPatternTitle; const size = 0; diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index 7455c4634deda..17f35967a626d 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -9,10 +9,10 @@ import { IScopedClusterClient } from 'kibana/server'; import { duration } from 'moment'; import { parseInterval } from '../../../common/util/parse_interval'; import { initCardinalityFieldsCache } from './fields_aggs_cache'; -import { DatafeedOverride } from '../../../common/types/modules'; import { AggCardinality } from '../../../common/types/fields'; import { isValidAggregationField } from '../../../common/util/validation_utils'; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; +import { Datafeed } from '../../../common/types/anomaly_detection_jobs'; /** * Service for carrying out queries to obtain data @@ -40,7 +40,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { async function getAggregatableFields( index: string | string[], fieldNames: string[], - datafeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ): Promise { const { body } = await asCurrentUser.fieldCaps({ index, @@ -87,7 +87,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { timeFieldName: string, earliestMs: number, latestMs: number, - datafeedConfig?: DatafeedOverride + datafeedConfig?: Datafeed ): Promise<{ [key: string]: number }> { const aggregatableFields = await getAggregatableFields(index, fieldNames, datafeedConfig); diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index 7ee292dba1325..769405c6ef7c2 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -7,7 +7,7 @@ import Boom from '@hapi/boom'; import { IScopedClusterClient } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; -import { AnalysisConfig } from '../../common/types/anomaly_detection_jobs'; +import { AnalysisConfig, Datafeed } from '../../common/types/anomaly_detection_jobs'; import { wrapError } from '../client/error_wrapper'; import { RouteInitialization } from '../types'; import { @@ -19,7 +19,6 @@ import { import { estimateBucketSpanFactory } from '../models/bucket_span_estimator'; import { calculateModelMemoryLimitProvider } from '../models/calculate_model_memory_limit'; import { validateJob, validateCardinality } from '../models/job_validation'; -import { DatafeedOverride } from '../../common/types/modules'; import type { MlClient } from '../lib/ml_client'; type CalculateModelMemoryLimitPayload = TypeOf; @@ -54,7 +53,7 @@ export function jobValidationRoutes( earliestMs, latestMs, undefined, - datafeedConfig as DatafeedOverride + datafeedConfig as Datafeed ); } From 1b5898f7ba953b6432d216268d14f71d817da48c Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 6 Nov 2020 10:55:08 -0600 Subject: [PATCH 20/22] [ML] Fix field in aggregatable check --- .../ml/server/models/job_validation/validate_cardinality.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts index c4473d7172ddc..f2bcc6e50d86e 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_cardinality.ts @@ -96,7 +96,7 @@ const validateFactory = (client: IScopedClusterClient, job: CombinedJob): Valida // if datafeed has aggregation fields, check recursively if field exist if ( datafeedAggregations !== undefined && - isValidAggregationField(datafeedAggregations, fieldName) + isValidAggregationField(datafeedAggregations, field) ) { return true; } From 22f938c9fd601a9c413d7f303ad0975997f649e7 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 11 Nov 2020 10:08:05 -0600 Subject: [PATCH 21/22] [ML] Update text description --- .../jobs/new_job/common/job_validator/util.ts | 3 +-- .../summary_count_field/description.tsx | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts index 239abedc352d8..04be935ed4399 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts @@ -198,8 +198,7 @@ export function populateValidationMessages( basicValidations.summaryCountField.message = i18n.translate( 'xpack.ml.newJob.wizard.validateJob.summaryCountFieldMissing', { - defaultMessage: - 'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.', + defaultMessage: 'Required field as the datafeed uses aggregations.', } ); } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx index 6e0c9124af1c2..a09b6540e101f 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/description.tsx @@ -7,8 +7,9 @@ import React, { memo, FC } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; +import { EuiDescribedFormGroup, EuiFormRow, EuiLink } from '@elastic/eui'; import { Validation } from '../../../../../common/job_validator'; +import { useMlKibana } from '../../../../../../../contexts/kibana'; interface Props { validation: Validation; @@ -18,13 +19,28 @@ export const Description: FC = memo(({ children, validation }) => { const title = i18n.translate('xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.title', { defaultMessage: 'Summary count field', }); + const { + services: { docLinks }, + } = useMlKibana(); + const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks; + const docsUrl = `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-aggregation.html`; return ( {title}} description={ + + + ), + }} /> } > From cf5d4d177c2672f1f336432072aa0777559fc42e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Wed, 11 Nov 2020 10:14:16 -0600 Subject: [PATCH 22/22] [ML] Update text translations --- x-pack/plugins/translations/translations/ja-JP.json | 13 ++++++------- x-pack/plugins/translations/translations/zh-CN.json | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 3f0db7fe5a99c..a55dfea79787b 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -1361,6 +1361,12 @@ "data.search.unableToGetSavedQueryToastTitle": "保存したクエリ {savedQueryId} を読み込めません", "data.search.upgradeLicense": "クエリがタイムアウトしました。無料のベーシックティアではクエリがタイムアウトすることはありません。", "data.search.upgradeLicenseActionText": "今すぐアップグレード", + "data.search.functions.kibana_context.filters.help": "Kibana ジェネリックフィルターを指定します", + "data.search.functions.kibana_context.help": "Kibana グローバルコンテキストを更新します", + "data.search.functions.kibana_context.q.help": "自由形式の Kibana テキストクエリを指定します", + "data.search.functions.kibana_context.savedSearchId.help": "クエリとフィルターに使用する保存検索ID を指定します。", + "data.search.functions.kibana_context.timeRange.help": "Kibana 時間範囲フィルターを指定します", + "data.search.functions.kibana.help": "Kibana グローバルコンテキストを取得します", "devTools.badge.readOnly.text": "読み込み専用", "devTools.badge.readOnly.tooltip": "を保存できませんでした", "devTools.devToolsTitle": "開発ツール", @@ -1662,12 +1668,6 @@ "expressions.functions.font.invalidFontWeightErrorMessage": "無効なフォント太さ：'{weight}'", "expressions.functions.font.invalidTextAlignmentErrorMessage": "無効なテキストアラインメント：'{align}'", "expressions.functions.fontHelpText": "フォントスタイルを作成します。", - "data.search.functions.kibana_context.filters.help": "Kibana ジェネリックフィルターを指定します", - "data.search.functions.kibana_context.help": "Kibana グローバルコンテキストを更新します", - "data.search.functions.kibana_context.q.help": "自由形式の Kibana テキストクエリを指定します", - "data.search.functions.kibana_context.savedSearchId.help": "クエリとフィルターに使用する保存検索ID を指定します。", - "data.search.functions.kibana_context.timeRange.help": "Kibana 時間範囲フィルターを指定します", - "data.search.functions.kibana.help": "Kibana グローバルコンテキストを取得します", "expressions.functions.theme.args.defaultHelpText": "テーマ情報がない場合のデフォルト値。", "expressions.functions.theme.args.variableHelpText": "読み取るテーマ変数名。", "expressions.functions.themeHelpText": "テーマ設定を読み取ります。", @@ -13216,7 +13216,6 @@ "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsErrorCallout": "停止したパーティションのリストの取得中にエラーが発生しました。", "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsExistCallout": "パーティション単位の分類とstop_on_warn設定が有効です。ジョブ「{jobId}」の一部のパーティションは分類に適さず、さらなる分類または異常検知分析から除外されました。", "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsPreviewColumnName": "停止したパーティション名", - "xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.description": "オプション。インプットデータが事前にまとめられている場合に使用、例: \\{docCountParam\\}。", "xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.title": "サマリーカウントフィールド", "xpack.ml.newJob.wizard.previewJsonButton": "JSON をプレビュー", "xpack.ml.newJob.wizard.previousStepButton": "前へ", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index e46cfb6658ea6..dff8a47b39902 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -1362,6 +1362,12 @@ "data.search.unableToGetSavedQueryToastTitle": "无法加载已保存查询 {savedQueryId}", "data.search.upgradeLicense": "您的查询已超时。使用我们免费的基础级许可，您的查询永不会超时。", "data.search.upgradeLicenseActionText": "立即升级", + "data.search.functions.kibana_context.filters.help": "指定 Kibana 常规筛选", + "data.search.functions.kibana_context.help": "更新 kibana 全局上下文", + "data.search.functions.kibana_context.q.help": "指定 Kibana 自由格式文本查询", + "data.search.functions.kibana_context.savedSearchId.help": "指定要用于查询和筛选的已保存搜索 ID", + "data.search.functions.kibana_context.timeRange.help": "指定 Kibana 时间范围筛选", + "data.search.functions.kibana.help": "获取 kibana 全局上下文", "devTools.badge.readOnly.text": "只读", "devTools.badge.readOnly.tooltip": "无法保存", "devTools.devToolsTitle": "开发工具", @@ -1663,12 +1669,6 @@ "expressions.functions.font.invalidFontWeightErrorMessage": "无效的字体粗细：“{weight}”", "expressions.functions.font.invalidTextAlignmentErrorMessage": "无效的文本对齐方式：“{align}”", "expressions.functions.fontHelpText": "创建字体样式。", - "data.search.functions.kibana_context.filters.help": "指定 Kibana 常规筛选", - "data.search.functions.kibana_context.help": "更新 kibana 全局上下文", - "data.search.functions.kibana_context.q.help": "指定 Kibana 自由格式文本查询", - "data.search.functions.kibana_context.savedSearchId.help": "指定要用于查询和筛选的已保存搜索 ID", - "data.search.functions.kibana_context.timeRange.help": "指定 Kibana 时间范围筛选", - "data.search.functions.kibana.help": "获取 kibana 全局上下文", "expressions.functions.theme.args.defaultHelpText": "主题信息不可用时的默认值。", "expressions.functions.theme.args.variableHelpText": "要读取的主题变量的名称。", "expressions.functions.themeHelpText": "读取主题设置。", @@ -13230,7 +13230,6 @@ "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsErrorCallout": "提取已停止分区的列表时发生错误。", "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsExistCallout": "启用按分区分类和 stop_on_warn 设置。作业“{jobId}”中的某些分区不适合进行分类，已从进一步分类或异常检测分析中排除。", "xpack.ml.newJob.wizard.pickFieldsStep.stoppedPartitionsPreviewColumnName": "已停止的分区名称", - "xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.description": "可选，用于输入数据已预汇总时，例如 \\{docCountParam\\}。", "xpack.ml.newJob.wizard.pickFieldsStep.summaryCountField.title": "汇总计数字段", "xpack.ml.newJob.wizard.previewJsonButton": "预览 JSON", "xpack.ml.newJob.wizard.previousStepButton": "上一页",