From 574205dc72b63a3c30ff159684012d8e3191ef2d Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 1 Oct 2020 18:14:59 +0100 Subject: [PATCH 1/3] chore(NA): remove non existing plugin paths from case api integration tests (#79127) * chore(NA): remove non existing plugin paths from case api integration tests config * chore(NA): remove unused import --- x-pack/test/case_api_integration/common/config.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/test/case_api_integration/common/config.ts b/x-pack/test/case_api_integration/common/config.ts index 5d34f8b04981a..72d1bc4ec9a37 100644 --- a/x-pack/test/case_api_integration/common/config.ts +++ b/x-pack/test/case_api_integration/common/config.ts @@ -4,8 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import path from 'path'; - import { CA_CERT_PATH } from '@kbn/dev-utils'; import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; @@ -78,8 +76,6 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`, '--xpack.eventLog.logEntries=true', ...disabledPlugins.map((key) => `--xpack.${key}.enabled=false`), - `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'alerts')}`, - `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'actions')}`, ...(ssl ? [ `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, From fd7dd41617a7464e6b0e61a83800c05ccf6a5189 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Thu, 1 Oct 2020 12:41:12 -0500 Subject: [PATCH 2/3] [ML] Update transform cloning to include description and new fields (#78364) --- .../public/app/common/request.test.ts | 33 +++++++++++++ .../transform/public/app/common/request.ts | 22 ++++++--- .../step_details/step_details_form.tsx | 18 +++++++ .../step_details/step_details_summary.tsx | 2 + .../test/functional/apps/transform/cloning.ts | 15 +++++- .../functional/services/transform/wizard.ts | 47 +++++++++++++++++++ 6 files changed, 129 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/transform/public/app/common/request.test.ts b/x-pack/plugins/transform/public/app/common/request.test.ts index 913ea8964eaf0..46ace2c3315a5 100644 --- a/x-pack/plugins/transform/public/app/common/request.test.ts +++ b/x-pack/plugins/transform/public/app/common/request.test.ts @@ -17,6 +17,7 @@ import { defaultQuery, getPreviewTransformRequestBody, getCreateTransformRequestBody, + getCreateTransformSettingsRequestBody, getPivotQuery, isDefaultQuery, isMatchAllQuery, @@ -159,6 +160,7 @@ describe('Transform: Common', () => { transformDescription: 'the-transform-description', transformFrequency: '1m', transformSettingsMaxPageSearchSize: 100, + transformSettingsDocsPerSecond: 400, destinationIndex: 'the-destination-index', touched: true, valid: true, @@ -180,6 +182,7 @@ describe('Transform: Common', () => { }, settings: { max_page_search_size: 100, + docs_per_second: 400, }, source: { index: ['the-index-pattern-title'], @@ -187,4 +190,34 @@ describe('Transform: Common', () => { }, }); }); + + test('getCreateTransformSettingsRequestBody() with multiple settings', () => { + const transformDetailsState: Partial = { + transformSettingsDocsPerSecond: 400, + transformSettingsMaxPageSearchSize: 100, + }; + + const request = getCreateTransformSettingsRequestBody(transformDetailsState); + + expect(request).toEqual({ + settings: { + docs_per_second: 400, + max_page_search_size: 100, + }, + }); + }); + + test('getCreateTransformSettingsRequestBody() with one setting', () => { + const transformDetailsState: Partial = { + transformSettingsDocsPerSecond: 400, + }; + + const request = getCreateTransformSettingsRequestBody(transformDetailsState); + + expect(request).toEqual({ + settings: { + docs_per_second: 400, + }, + }); + }); }); diff --git a/x-pack/plugins/transform/public/app/common/request.ts b/x-pack/plugins/transform/public/app/common/request.ts index 45160d125309d..8ee235baf7c5a 100644 --- a/x-pack/plugins/transform/public/app/common/request.ts +++ b/x-pack/plugins/transform/public/app/common/request.ts @@ -130,6 +130,20 @@ export function getPreviewTransformRequestBody( return request; } +export const getCreateTransformSettingsRequestBody = ( + transformDetailsState: Partial +): { settings?: PutTransformsRequestSchema['settings'] } => { + const settings: PutTransformsRequestSchema['settings'] = { + ...(transformDetailsState.transformSettingsMaxPageSearchSize + ? { max_page_search_size: transformDetailsState.transformSettingsMaxPageSearchSize } + : {}), + ...(transformDetailsState.transformSettingsDocsPerSecond + ? { docs_per_second: transformDetailsState.transformSettingsDocsPerSecond } + : {}), + }; + return Object.keys(settings).length > 0 ? { settings } : {}; +}; + export const getCreateTransformRequestBody = ( indexPatternTitle: IndexPattern['title'], pivotState: StepDefineExposedState, @@ -164,13 +178,7 @@ export const getCreateTransformRequestBody = ( } : {}), // conditionally add additional settings - ...(transformDetailsState.transformSettingsMaxPageSearchSize - ? { - settings: { - max_page_search_size: transformDetailsState.transformSettingsMaxPageSearchSize, - }, - } - : {}), + ...getCreateTransformSettingsRequestBody(transformDetailsState), }); export function isHttpFetchError(error: any): error is HttpFetchError { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx index 00ab516f625fe..9b43879512e4d 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx @@ -63,6 +63,7 @@ export interface StepDetailsExposedState { transformDescription: string; transformFrequency: string; transformSettingsMaxPageSearchSize: number; + transformSettingsDocsPerSecond?: number; valid: boolean; indexPatternTimeField?: string | undefined; } @@ -100,6 +101,20 @@ export function applyTransformConfigToDetailsState( state.continuousModeDelay = time?.delay ?? defaultContinuousModeDelay; state.isContinuousModeEnabled = true; } + if (transformConfig.description !== undefined) { + state.transformDescription = transformConfig.description; + } + if (transformConfig.frequency !== undefined) { + state.transformFrequency = transformConfig.frequency; + } + if (transformConfig.settings) { + if (typeof transformConfig.settings?.max_page_search_size === 'number') { + state.transformSettingsMaxPageSearchSize = transformConfig.settings.max_page_search_size; + } + if (typeof transformConfig.settings?.docs_per_second === 'number') { + state.transformSettingsDocsPerSecond = transformConfig.settings.docs_per_second; + } + } } return state; } @@ -275,6 +290,8 @@ export const StepDetailsForm: FC = React.memo( const [transformSettingsMaxPageSearchSize, setTransformSettingsMaxPageSearchSize] = useState( defaults.transformSettingsMaxPageSearchSize ); + const [transformSettingsDocsPerSecond] = useState(defaults.transformSettingsDocsPerSecond); + const isTransformSettingsMaxPageSearchSizeValid = transformSettingsMaxPageSearchSizeValidator( transformSettingsMaxPageSearchSize ); @@ -301,6 +318,7 @@ export const StepDetailsForm: FC = React.memo( transformDescription, transformFrequency, transformSettingsMaxPageSearchSize, + transformSettingsDocsPerSecond, destinationIndex, touched: true, valid, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx index 45cd8aa465522..f5444eaf6640a 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_summary.tsx @@ -98,6 +98,7 @@ export const StepDetailsSummary: FC = React.memo((props paddingSize="s" > = React.memo((props {transformFrequency} Date: Thu, 1 Oct 2020 12:42:37 -0500 Subject: [PATCH 3/3] Revert "[Metrics UI] Add ability to override datafeeds and job config for partition field (#78875)" This reverts commit ee7672aaf074dc4ebaf0ffb88d95d5f1bf9e1d18. --- .../containers/ml/infra_ml_module_types.ts | 4 +- .../containers/ml/infra_ml_setup_state.ts | 289 ++++++++++++++++++ .../metrics_hosts/module_descriptor.ts | 135 +++----- .../modules/metrics_k8s/module_descriptor.ts | 143 +++------ .../anomoly_detection_flyout.tsx | 4 +- .../ml/anomaly_detection/flyout_home.tsx | 113 ++++--- .../ml/anomaly_detection/job_setup_screen.tsx | 3 +- 7 files changed, 444 insertions(+), 247 deletions(-) create mode 100644 x-pack/plugins/infra/public/containers/ml/infra_ml_setup_state.ts diff --git a/x-pack/plugins/infra/public/containers/ml/infra_ml_module_types.ts b/x-pack/plugins/infra/public/containers/ml/infra_ml_module_types.ts index e36f38add641a..a9f2671de8259 100644 --- a/x-pack/plugins/infra/public/containers/ml/infra_ml_module_types.ts +++ b/x-pack/plugins/infra/public/containers/ml/infra_ml_module_types.ts @@ -33,11 +33,11 @@ export interface ModuleDescriptor { partitionField?: string ) => Promise; cleanUpModule: (spaceId: string, sourceId: string) => Promise; - validateSetupIndices?: ( + validateSetupIndices: ( indices: string[], timestampField: string ) => Promise; - validateSetupDatasets?: ( + validateSetupDatasets: ( indices: string[], timestampField: string, startTime: number, diff --git a/x-pack/plugins/infra/public/containers/ml/infra_ml_setup_state.ts b/x-pack/plugins/infra/public/containers/ml/infra_ml_setup_state.ts new file mode 100644 index 0000000000000..0dfe3b301f240 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/ml/infra_ml_setup_state.ts @@ -0,0 +1,289 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEqual } from 'lodash'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { usePrevious } from 'react-use'; +import { + combineDatasetFilters, + DatasetFilter, + filterDatasetFilter, + isExampleDataIndex, +} from '../../../common/infra_ml'; +import { + AvailableIndex, + ValidationIndicesError, + ValidationUIError, +} from '../../components/logging/log_analysis_setup/initial_configuration_step'; +import { useTrackedPromise } from '../../utils/use_tracked_promise'; +import { ModuleDescriptor, ModuleSourceConfiguration } from './infra_ml_module_types'; + +type SetupHandler = ( + indices: string[], + startTime: number | undefined, + endTime: number | undefined, + datasetFilter: DatasetFilter +) => void; + +interface AnalysisSetupStateArguments { + cleanUpAndSetUpModule: SetupHandler; + moduleDescriptor: ModuleDescriptor; + setUpModule: SetupHandler; + sourceConfiguration: ModuleSourceConfiguration; +} + +const fourWeeksInMs = 86400000 * 7 * 4; + +export const useAnalysisSetupState = ({ + cleanUpAndSetUpModule, + moduleDescriptor: { validateSetupDatasets, validateSetupIndices }, + setUpModule, + sourceConfiguration, +}: AnalysisSetupStateArguments) => { + const [startTime, setStartTime] = useState(Date.now() - fourWeeksInMs); + const [endTime, setEndTime] = useState(undefined); + + const isTimeRangeValid = useMemo( + () => (startTime != null && endTime != null ? startTime < endTime : true), + [endTime, startTime] + ); + + const [validatedIndices, setValidatedIndices] = useState( + sourceConfiguration.indices.map((indexName) => ({ + name: indexName, + validity: 'unknown' as const, + })) + ); + + const updateIndicesWithValidationErrors = useCallback( + (validationErrors: ValidationIndicesError[]) => + setValidatedIndices((availableIndices) => + availableIndices.map((previousAvailableIndex) => { + const indexValiationErrors = validationErrors.filter( + ({ index }) => index === previousAvailableIndex.name + ); + + if (indexValiationErrors.length > 0) { + return { + validity: 'invalid', + name: previousAvailableIndex.name, + errors: indexValiationErrors, + }; + } else if (previousAvailableIndex.validity === 'valid') { + return { + ...previousAvailableIndex, + validity: 'valid', + errors: [], + }; + } else { + return { + validity: 'valid', + name: previousAvailableIndex.name, + isSelected: !isExampleDataIndex(previousAvailableIndex.name), + availableDatasets: [], + datasetFilter: { + type: 'includeAll' as const, + }, + }; + } + }) + ), + [] + ); + + const updateIndicesWithAvailableDatasets = useCallback( + (availableDatasets: Array<{ indexName: string; datasets: string[] }>) => + setValidatedIndices((availableIndices) => + availableIndices.map((previousAvailableIndex) => { + if (previousAvailableIndex.validity !== 'valid') { + return previousAvailableIndex; + } + + const availableDatasetsForIndex = availableDatasets.filter( + ({ indexName }) => indexName === previousAvailableIndex.name + ); + const newAvailableDatasets = availableDatasetsForIndex.flatMap( + ({ datasets }) => datasets + ); + + // filter out datasets that have disappeared if this index' datasets were updated + const newDatasetFilter: DatasetFilter = + availableDatasetsForIndex.length > 0 + ? filterDatasetFilter(previousAvailableIndex.datasetFilter, (dataset) => + newAvailableDatasets.includes(dataset) + ) + : previousAvailableIndex.datasetFilter; + + return { + ...previousAvailableIndex, + availableDatasets: newAvailableDatasets, + datasetFilter: newDatasetFilter, + }; + }) + ), + [] + ); + + const validIndexNames = useMemo( + () => validatedIndices.filter((index) => index.validity === 'valid').map((index) => index.name), + [validatedIndices] + ); + + const selectedIndexNames = useMemo( + () => + validatedIndices + .filter((index) => index.validity === 'valid' && index.isSelected) + .map((i) => i.name), + [validatedIndices] + ); + + const datasetFilter = useMemo( + () => + validatedIndices + .flatMap((validatedIndex) => + validatedIndex.validity === 'valid' + ? validatedIndex.datasetFilter + : { type: 'includeAll' as const } + ) + .reduce(combineDatasetFilters, { type: 'includeAll' as const }), + [validatedIndices] + ); + + const [validateIndicesRequest, validateIndices] = useTrackedPromise( + { + cancelPreviousOn: 'resolution', + createPromise: async () => { + return await validateSetupIndices( + sourceConfiguration.indices, + sourceConfiguration.timestampField + ); + }, + onResolve: ({ data: { errors } }) => { + updateIndicesWithValidationErrors(errors); + }, + onReject: () => { + setValidatedIndices([]); + }, + }, + [sourceConfiguration.indices, sourceConfiguration.timestampField] + ); + + const [validateDatasetsRequest, validateDatasets] = useTrackedPromise( + { + cancelPreviousOn: 'resolution', + createPromise: async () => { + if (validIndexNames.length === 0) { + return { data: { datasets: [] } }; + } + + return await validateSetupDatasets( + validIndexNames, + sourceConfiguration.timestampField, + startTime ?? 0, + endTime ?? Date.now() + ); + }, + onResolve: ({ data: { datasets } }) => { + updateIndicesWithAvailableDatasets(datasets); + }, + }, + [validIndexNames, sourceConfiguration.timestampField, startTime, endTime] + ); + + const setUp = useCallback(() => { + return setUpModule(selectedIndexNames, startTime, endTime, datasetFilter); + }, [setUpModule, selectedIndexNames, startTime, endTime, datasetFilter]); + + const cleanUpAndSetUp = useCallback(() => { + return cleanUpAndSetUpModule(selectedIndexNames, startTime, endTime, datasetFilter); + }, [cleanUpAndSetUpModule, selectedIndexNames, startTime, endTime, datasetFilter]); + + const isValidating = useMemo( + () => validateIndicesRequest.state === 'pending' || validateDatasetsRequest.state === 'pending', + [validateDatasetsRequest.state, validateIndicesRequest.state] + ); + + const validationErrors = useMemo(() => { + if (isValidating) { + return []; + } + + return [ + // validate request status + ...(validateIndicesRequest.state === 'rejected' || + validateDatasetsRequest.state === 'rejected' + ? [{ error: 'NETWORK_ERROR' as const }] + : []), + // validation request results + ...validatedIndices.reduce((errors, index) => { + return index.validity === 'invalid' && selectedIndexNames.includes(index.name) + ? [...errors, ...index.errors] + : errors; + }, []), + // index count + ...(selectedIndexNames.length === 0 ? [{ error: 'TOO_FEW_SELECTED_INDICES' as const }] : []), + // time range + ...(!isTimeRangeValid ? [{ error: 'INVALID_TIME_RANGE' as const }] : []), + ]; + }, [ + isValidating, + validateIndicesRequest.state, + validateDatasetsRequest.state, + validatedIndices, + selectedIndexNames, + isTimeRangeValid, + ]); + + const prevStartTime = usePrevious(startTime); + const prevEndTime = usePrevious(endTime); + const prevValidIndexNames = usePrevious(validIndexNames); + + useEffect(() => { + if (!isTimeRangeValid) { + return; + } + + validateIndices(); + }, [isTimeRangeValid, validateIndices]); + + useEffect(() => { + if (!isTimeRangeValid) { + return; + } + + if ( + startTime !== prevStartTime || + endTime !== prevEndTime || + !isEqual(validIndexNames, prevValidIndexNames) + ) { + validateDatasets(); + } + }, [ + endTime, + isTimeRangeValid, + prevEndTime, + prevStartTime, + prevValidIndexNames, + startTime, + validIndexNames, + validateDatasets, + ]); + + return { + cleanUpAndSetUp, + datasetFilter, + endTime, + isValidating, + selectedIndexNames, + setEndTime, + setStartTime, + setUp, + startTime, + validatedIndices, + setValidatedIndices, + validationErrors, + }; +}; diff --git a/x-pack/plugins/infra/public/containers/ml/modules/metrics_hosts/module_descriptor.ts b/x-pack/plugins/infra/public/containers/ml/modules/metrics_hosts/module_descriptor.ts index 711ee76d42a64..cec87fb1144e3 100644 --- a/x-pack/plugins/infra/public/containers/ml/modules/metrics_hosts/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/ml/modules/metrics_hosts/module_descriptor.ts @@ -10,27 +10,17 @@ import { cleanUpJobsAndDatafeeds } from '../../infra_ml_cleanup'; import { callJobsSummaryAPI } from '../../api/ml_get_jobs_summary_api'; import { callGetMlModuleAPI } from '../../api/ml_get_module'; import { callSetupMlModuleAPI } from '../../api/ml_setup_module_api'; +import { callValidateIndicesAPI } from '../../../logs/log_analysis/api/validate_indices'; +import { callValidateDatasetsAPI } from '../../../logs/log_analysis/api/validate_datasets'; import { metricsHostsJobTypes, getJobId, MetricsHostsJobType, DatasetFilter, bucketSpan, + partitionField, } from '../../../../../common/infra_ml'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import MemoryJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import MemoryDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkInJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkInDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkOutJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkOutDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json'; -type JobType = 'hosts_memory_usage' | 'hosts_network_in' | 'hosts_network_out'; const moduleId = 'metrics_ui_hosts'; const moduleName = i18n.translate('xpack.infra.ml.metricsModuleName', { defaultMessage: 'Metrics anomanly detection', @@ -64,68 +54,23 @@ const setUpModule = async ( end: number | undefined, datasetFilter: DatasetFilter, { spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration, - partitionField?: string + pField?: string ) => { const indexNamePattern = indices.join(','); - const jobIds: JobType[] = ['hosts_memory_usage', 'hosts_network_in', 'hosts_network_out']; - - const jobOverrides = jobIds.map((id) => { - const { job: defaultJobConfig } = getDefaultJobConfigs(id); - - // eslint-disable-next-line @typescript-eslint/naming-convention - const analysis_config = { - ...defaultJobConfig.analysis_config, - }; - - if (partitionField) { - analysis_config.detectors[0].partition_field_name = partitionField; - if (analysis_config.influencers.indexOf(partitionField) === -1) { - analysis_config.influencers.push(partitionField); - } - } - - return { - job_id: id, - data_description: { - time_field: timestampField, - }, - analysis_config, - custom_settings: { - metrics_source_config: { - indexPattern: indexNamePattern, - timestampField, - bucketSpan, - }, - }, - }; - }); - - const datafeedOverrides = jobIds.map((id) => { - const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id); - - if (!partitionField || id === 'hosts_memory_usage') { - // Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field - return defaultDatafeedConfig; - } - - // If we have a partition field, we need to change the aggregation to do a terms agg at the top level - const aggregations = { - [partitionField]: { - terms: { - field: partitionField, - }, - aggregations: { - ...defaultDatafeedConfig.aggregations, - }, + const jobIds = ['hosts_memory_usage', 'hosts_network_in', 'hosts_network_out']; + const jobOverrides = jobIds.map((id) => ({ + job_id: id, + data_description: { + time_field: timestampField, + }, + custom_settings: { + metrics_source_config: { + indexPattern: indexNamePattern, + timestampField, + bucketSpan, }, - }; - - return { - ...defaultDatafeedConfig, - job_id: id, - aggregations, - }; - }); + }, + })); return callSetupMlModuleAPI( moduleId, @@ -135,34 +80,36 @@ const setUpModule = async ( sourceId, indexNamePattern, jobOverrides, - datafeedOverrides + [] ); }; -const getDefaultJobConfigs = (jobId: JobType) => { - switch (jobId) { - case 'hosts_memory_usage': - return { - datafeed: MemoryDatafeed, - job: MemoryJob, - }; - case 'hosts_network_in': - return { - datafeed: NetworkInDatafeed, - job: NetworkInJob, - }; - case 'hosts_network_out': - return { - datafeed: NetworkOutDatafeed, - job: NetworkOutJob, - }; - } -}; - const cleanUpModule = async (spaceId: string, sourceId: string) => { return await cleanUpJobsAndDatafeeds(spaceId, sourceId, metricsHostsJobTypes); }; +const validateSetupIndices = async (indices: string[], timestampField: string) => { + return await callValidateIndicesAPI(indices, [ + { + name: timestampField, + validTypes: ['date'], + }, + { + name: partitionField, + validTypes: ['keyword'], + }, + ]); +}; + +const validateSetupDatasets = async ( + indices: string[], + timestampField: string, + startTime: number, + endTime: number +) => { + return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime); +}; + export const metricHostsModule: ModuleDescriptor = { moduleId, moduleName, @@ -174,4 +121,6 @@ export const metricHostsModule: ModuleDescriptor = { getModuleDefinition, setUpModule, cleanUpModule, + validateSetupDatasets, + validateSetupIndices, }; diff --git a/x-pack/plugins/infra/public/containers/ml/modules/metrics_k8s/module_descriptor.ts b/x-pack/plugins/infra/public/containers/ml/modules/metrics_k8s/module_descriptor.ts index 41c6df92fb379..cbcff1c307af6 100644 --- a/x-pack/plugins/infra/public/containers/ml/modules/metrics_k8s/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/ml/modules/metrics_k8s/module_descriptor.ts @@ -10,28 +10,17 @@ import { cleanUpJobsAndDatafeeds } from '../../infra_ml_cleanup'; import { callJobsSummaryAPI } from '../../api/ml_get_jobs_summary_api'; import { callGetMlModuleAPI } from '../../api/ml_get_module'; import { callSetupMlModuleAPI } from '../../api/ml_setup_module_api'; +import { callValidateIndicesAPI } from '../../../logs/log_analysis/api/validate_indices'; +import { callValidateDatasetsAPI } from '../../../logs/log_analysis/api/validate_datasets'; import { metricsK8SJobTypes, getJobId, MetricK8sJobType, DatasetFilter, bucketSpan, + partitionField, } from '../../../../../common/infra_ml'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import MemoryJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import MemoryDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkInJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkInDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkOutJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import NetworkOutDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json'; -type JobType = 'k8s_memory_usage' | 'k8s_network_in' | 'k8s_network_out'; -export const DEFAULT_K8S_PARTITION_FIELD = 'kubernetes.namespace'; const moduleId = 'metrics_ui_k8s'; const moduleName = i18n.translate('xpack.infra.ml.metricsModuleName', { defaultMessage: 'Metrics anomanly detection', @@ -65,72 +54,26 @@ const setUpModule = async ( end: number | undefined, datasetFilter: DatasetFilter, { spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration, - partitionField?: string + pField?: string ) => { const indexNamePattern = indices.join(','); - const jobIds: JobType[] = ['k8s_memory_usage', 'k8s_network_in', 'k8s_network_out']; - const jobOverrides = jobIds.map((id) => { - const { job: defaultJobConfig } = getDefaultJobConfigs(id); - - // eslint-disable-next-line @typescript-eslint/naming-convention - const analysis_config = { - ...defaultJobConfig.analysis_config, - }; - - if (partitionField) { - analysis_config.detectors[0].partition_field_name = partitionField; - if (analysis_config.influencers.indexOf(partitionField) === -1) { - analysis_config.influencers.push(partitionField); - } - } - - return { - job_id: id, - data_description: { - time_field: timestampField, - }, - analysis_config, - custom_settings: { - metrics_source_config: { - indexPattern: indexNamePattern, - timestampField, - bucketSpan, - }, - }, - }; - }); - - const datafeedOverrides = jobIds.map((id) => { - const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id); - - if (!partitionField || id === 'k8s_memory_usage') { - // Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field - return defaultDatafeedConfig; - } - - // Because the ML K8s jobs ship with a default partition field of {kubernetes.namespace}, ignore that agg and wrap it in our own agg. - const innerAggregation = - defaultDatafeedConfig.aggregations[DEFAULT_K8S_PARTITION_FIELD].aggregations; - - // If we have a partition field, we need to change the aggregation to do a terms agg to partition the data at the top level - const aggregations = { - [partitionField]: { - terms: { - field: partitionField, - size: 25, // 25 is arbitratry and only used to keep the number of buckets to a managable level in the event that the user choose a high cardinality partition field. - }, - aggregations: { - ...innerAggregation, - }, + const jobIds = ['k8s_memory_usage', 'k8s_network_in', 'k8s_network_out']; + const jobOverrides = jobIds.map((id) => ({ + job_id: id, + analysis_config: { + bucket_span: `${bucketSpan}ms`, + }, + data_description: { + time_field: timestampField, + }, + custom_settings: { + metrics_source_config: { + indexPattern: indexNamePattern, + timestampField, + bucketSpan, }, - }; - - return { - ...defaultDatafeedConfig, - job_id: id, - aggregations, - }; - }); + }, + })); return callSetupMlModuleAPI( moduleId, @@ -140,34 +83,36 @@ const setUpModule = async ( sourceId, indexNamePattern, jobOverrides, - datafeedOverrides + [] ); }; -const getDefaultJobConfigs = (jobId: JobType) => { - switch (jobId) { - case 'k8s_memory_usage': - return { - datafeed: MemoryDatafeed, - job: MemoryJob, - }; - case 'k8s_network_in': - return { - datafeed: NetworkInDatafeed, - job: NetworkInJob, - }; - case 'k8s_network_out': - return { - datafeed: NetworkOutDatafeed, - job: NetworkOutJob, - }; - } -}; - const cleanUpModule = async (spaceId: string, sourceId: string) => { return await cleanUpJobsAndDatafeeds(spaceId, sourceId, metricsK8SJobTypes); }; +const validateSetupIndices = async (indices: string[], timestampField: string) => { + return await callValidateIndicesAPI(indices, [ + { + name: timestampField, + validTypes: ['date'], + }, + { + name: partitionField, + validTypes: ['keyword'], + }, + ]); +}; + +const validateSetupDatasets = async ( + indices: string[], + timestampField: string, + startTime: number, + endTime: number +) => { + return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime); +}; + export const metricHostsModule: ModuleDescriptor = { moduleId, moduleName, @@ -179,4 +124,6 @@ export const metricHostsModule: ModuleDescriptor = { getModuleDefinition, setUpModule, cleanUpModule, + validateSetupDatasets, + validateSetupIndices, }; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/anomoly_detection_flyout.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/anomoly_detection_flyout.tsx index b5d224910e819..b063713fa2c97 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/anomoly_detection_flyout.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/anomoly_detection_flyout.tsx @@ -50,10 +50,10 @@ export const AnomalyDetectionFlyout = () => { return ( <> - + {showFlyout && ( diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx index 5b520084ebb74..801dff9c4a17a 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx @@ -5,7 +5,7 @@ */ import React, { useState, useCallback, useEffect } from 'react'; -import { EuiFlyoutHeader, EuiTitle, EuiFlyoutBody, EuiSpacer } from '@elastic/eui'; +import { EuiFlyoutHeader, EuiTitle, EuiFlyoutBody, EuiTabs, EuiTab, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText, EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -30,7 +30,7 @@ interface Props { } export const FlyoutHome = (props: Props) => { - const [tab] = useState<'jobs' | 'anomalies'>('jobs'); + const [tab, setTab] = useState<'jobs' | 'anomalies'>('jobs'); const { goToSetup } = props; const { fetchJobStatus: fetchHostJobStatus, @@ -56,10 +56,18 @@ export const FlyoutHome = (props: Props) => { goToSetup('kubernetes'); }, [goToSetup]); + const goToJobs = useCallback(() => { + setTab('jobs'); + }, []); + const jobIds = [ ...(k8sJobSummaries || []).map((k) => k.id), ...(hostJobSummaries || []).map((h) => h.id), ]; + const anomaliesUrl = useLinkProps({ + app: 'ml', + pathname: `/explorer?_g=${createResultsUrl(jobIds)}`, + }); useEffect(() => { if (hasInfraMLReadCapabilities) { @@ -97,24 +105,30 @@ export const FlyoutHome = (props: Props) => { -
- -

- -

- -
- + + + + + + + + {hostJobSummaries.length > 0 && ( <> 0} hasK8sJobs={k8sJobSummaries.length > 0} - jobIds={jobIds} /> @@ -137,7 +151,6 @@ export const FlyoutHome = (props: Props) => { interface CalloutProps { hasHostJobs: boolean; hasK8sJobs: boolean; - jobIds: string[]; } const JobsEnabledCallout = (props: CalloutProps) => { let target = ''; @@ -162,34 +175,8 @@ const JobsEnabledCallout = (props: CalloutProps) => { pathname: '/jobs', }); - const anomaliesUrl = useLinkProps({ - app: 'ml', - pathname: `/explorer?_g=${createResultsUrl(props.jobIds)}`, - }); - return ( <> - - - - - - - - - - - - - - - { } iconType="check" /> + + + + ); }; @@ -217,11 +211,30 @@ interface CreateJobTab { const CreateJobTab = (props: CreateJobTab) => { return ( <> - {/* */} +
+ +

+ +

+ + +

+ +

+ +
+ + } // title="Hosts" title={ @@ -232,7 +245,7 @@ const CreateJobTab = (props: CreateJobTab) => { } description={ } @@ -241,7 +254,7 @@ const CreateJobTab = (props: CreateJobTab) => { {props.hasHostJobs && ( @@ -249,7 +262,7 @@ const CreateJobTab = (props: CreateJobTab) => { {!props.hasHostJobs && ( @@ -260,7 +273,7 @@ const CreateJobTab = (props: CreateJobTab) => { } title={ { } description={ } @@ -279,7 +292,7 @@ const CreateJobTab = (props: CreateJobTab) => { {props.hasK8sJobs && ( @@ -287,7 +300,7 @@ const CreateJobTab = (props: CreateJobTab) => { {!props.hasK8sJobs && ( diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx index c327d187f6bc2..428c002da6383 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx @@ -20,7 +20,6 @@ import { useSourceViaHttp } from '../../../../../../containers/source/use_source import { useMetricK8sModuleContext } from '../../../../../../containers/ml/modules/metrics_k8s/module'; import { useMetricHostsModuleContext } from '../../../../../../containers/ml/modules/metrics_hosts/module'; import { FixedDatePicker } from '../../../../../../components/fixed_datepicker'; -import { DEFAULT_K8S_PARTITION_FIELD } from '../../../../../../containers/ml/modules/metrics_k8s/module_descriptor'; interface Props { jobType: 'hosts' | 'kubernetes'; @@ -108,7 +107,7 @@ export const JobSetupScreen = (props: Props) => { useEffect(() => { if (props.jobType === 'kubernetes') { - setPartitionField([DEFAULT_K8S_PARTITION_FIELD]); + setPartitionField(['kubernetes.namespace']); } }, [props.jobType]);