From cae18314b28204cc79ea522fa0a1190422741dda Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Mon, 1 Jun 2020 23:05:44 -0500 Subject: [PATCH] [7.x] [ML] Add ability to delete target index & index pattern when deleting DFA job (#66934) (#67875) Co-authored-by: Elastic Machine --- .../ml/common/types/data_frame_analytics.ts | 11 + .../analytics_list/action_delete.test.tsx | 51 +++- .../analytics_list/action_delete.tsx | 150 +++++++++++- .../analytics_service/delete_analytics.ts | 131 ++++++++++- .../services/analytics_service/index.ts | 3 +- .../ml_api_service/data_frame_analytics.ts | 19 ++ .../services/ml_api_service/jobs.ts | 1 - .../ml/public/application/util/error_utils.ts | 32 +++ .../data_frame_analytics/index_patterns.ts | 32 +++ .../ml/server/routes/data_frame_analytics.ts | 112 ++++++++- .../routes/schemas/data_analytics_schema.ts | 8 + .../apis/ml/data_frame_analytics/delete.ts | 218 ++++++++++++++++++ .../apis/ml/data_frame_analytics/index.ts | 1 + x-pack/test/functional/services/ml/api.ts | 53 ++++- .../functional/services/ml/test_resources.ts | 17 +- 15 files changed, 797 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugins/ml/common/types/data_frame_analytics.ts create mode 100644 x-pack/plugins/ml/public/application/util/error_utils.ts create mode 100644 x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts create mode 100644 x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts new file mode 100644 index 00000000000000..5ba7f9c191a7fe --- /dev/null +++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CustomHttpResponseOptions, ResponseError } from 'kibana/server'; +export interface DeleteDataFrameAnalyticsWithIndexStatus { + success: boolean; + error?: CustomHttpResponseOptions; +} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.test.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.test.tsx index 2ef1515726d1b3..33217f127f9982 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.test.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.test.tsx @@ -5,20 +5,41 @@ */ import React from 'react'; -import { render } from '@testing-library/react'; - +import { fireEvent, render } from '@testing-library/react'; import * as CheckPrivilige from '../../../../../capabilities/check_capabilities'; - -import { DeleteAction } from './action_delete'; - import mockAnalyticsListItem from './__mocks__/analytics_list_item.json'; +import { DeleteAction } from './action_delete'; +import { I18nProvider } from '@kbn/i18n/react'; +import { + coreMock as mockCoreServices, + i18nServiceMock, +} from '../../../../../../../../../../src/core/public/mocks'; jest.mock('../../../../../capabilities/check_capabilities', () => ({ checkPermission: jest.fn(() => false), createPermissionFailureMessage: jest.fn(), })); +jest.mock('../../../../../../application/util/dependency_cache', () => ({ + getToastNotifications: () => ({ addSuccess: jest.fn(), addDanger: jest.fn() }), +})); + +jest.mock('../../../../../contexts/kibana', () => ({ + useMlKibana: () => ({ + services: mockCoreServices.createStart(), + }), +})); +export const MockI18nService = i18nServiceMock.create(); +export const I18nServiceConstructor = jest.fn().mockImplementation(() => MockI18nService); +jest.doMock('@kbn/i18n', () => ({ + I18nService: I18nServiceConstructor, +})); + describe('DeleteAction', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + test('When canDeleteDataFrameAnalytics permission is false, button should be disabled.', () => { const { getByTestId } = render(); expect(getByTestId('mlAnalyticsJobDeleteButton')).toHaveAttribute('disabled'); @@ -46,4 +67,24 @@ describe('DeleteAction', () => { expect(getByTestId('mlAnalyticsJobDeleteButton')).toHaveAttribute('disabled'); }); + + describe('When delete model is open', () => { + test('should allow to delete target index by default.', () => { + const mock = jest.spyOn(CheckPrivilige, 'checkPermission'); + mock.mockImplementation((p) => p === 'canDeleteDataFrameAnalytics'); + const { getByTestId, queryByTestId } = render( + + + + ); + const deleteButton = getByTestId('mlAnalyticsJobDeleteButton'); + fireEvent.click(deleteButton); + expect(getByTestId('mlAnalyticsJobDeleteModal')).toBeInTheDocument(); + expect(getByTestId('mlAnalyticsJobDeleteIndexSwitch')).toBeInTheDocument(); + const mlAnalyticsJobDeleteIndexSwitch = getByTestId('mlAnalyticsJobDeleteIndexSwitch'); + expect(mlAnalyticsJobDeleteIndexSwitch).toHaveAttribute('aria-checked', 'true'); + expect(queryByTestId('mlAnalyticsJobDeleteIndexPatternSwitch')).toBeNull(); + mock.mockRestore(); + }); + }); }); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.tsx index 2923938ae68ace..2d433f6b184846 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/action_delete.tsx @@ -4,24 +4,32 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useState } from 'react'; +import React, { Fragment, FC, useState, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiButtonEmpty, EuiConfirmModal, EuiOverlayMask, EuiToolTip, + EuiSwitch, + EuiFlexGroup, + EuiFlexItem, EUI_MODAL_CONFIRM_BUTTON, } from '@elastic/eui'; - -import { deleteAnalytics } from '../../services/analytics_service'; - +import { IIndexPattern } from 'src/plugins/data/common'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + deleteAnalytics, + deleteAnalyticsAndDestIndex, + canDeleteIndex, +} from '../../services/analytics_service'; import { checkPermission, createPermissionFailureMessage, } from '../../../../../capabilities/check_capabilities'; - +import { useMlKibana } from '../../../../../contexts/kibana'; import { isDataFrameAnalyticsRunning, DataFrameAnalyticsListRow } from './common'; +import { extractErrorMessage } from '../../../../../util/error_utils'; interface DeleteActionProps { item: DataFrameAnalyticsListRow; @@ -29,17 +37,99 @@ interface DeleteActionProps { export const DeleteAction: FC = ({ item }) => { const disabled = isDataFrameAnalyticsRunning(item.stats.state); - const canDeleteDataFrameAnalytics: boolean = checkPermission('canDeleteDataFrameAnalytics'); const [isModalVisible, setModalVisible] = useState(false); + const [deleteTargetIndex, setDeleteTargetIndex] = useState(true); + const [deleteIndexPattern, setDeleteIndexPattern] = useState(true); + const [userCanDeleteIndex, setUserCanDeleteIndex] = useState(false); + const [indexPatternExists, setIndexPatternExists] = useState(false); + + const { savedObjects, notifications } = useMlKibana().services; + const savedObjectsClient = savedObjects.client; + + const indexName = item.config.dest.index; + + const checkIndexPatternExists = async () => { + try { + const response = await savedObjectsClient.find({ + type: 'index-pattern', + perPage: 10, + search: `"${indexName}"`, + searchFields: ['title'], + fields: ['title'], + }); + const ip = response.savedObjects.find( + (obj) => obj.attributes.title.toLowerCase() === indexName.toLowerCase() + ); + if (ip !== undefined) { + setIndexPatternExists(true); + } + } catch (e) { + const { toasts } = notifications; + const error = extractErrorMessage(e); + + toasts.addDanger( + i18n.translate( + 'xpack.ml.dataframe.analyticsList.errorWithCheckingIfIndexPatternExistsNotificationErrorMessage', + { + defaultMessage: + 'An error occurred checking if index pattern {indexPattern} exists: {error}', + values: { indexPattern: indexName, error }, + } + ) + ); + } + }; + const checkUserIndexPermission = () => { + try { + const userCanDelete = canDeleteIndex(indexName); + if (userCanDelete) { + setUserCanDeleteIndex(true); + } + } catch (e) { + const { toasts } = notifications; + const error = extractErrorMessage(e); + + toasts.addDanger( + i18n.translate( + 'xpack.ml.dataframe.analyticsList.errorWithCheckingIfUserCanDeleteIndexNotificationErrorMessage', + { + defaultMessage: + 'An error occurred checking if user can delete {destinationIndex}: {error}', + values: { destinationIndex: indexName, error }, + } + ) + ); + } + }; + + useEffect(() => { + // Check if an index pattern exists corresponding to current DFA job + // if pattern does exist, show it to user + checkIndexPatternExists(); + + // Check if an user has permission to delete the index & index pattern + checkUserIndexPermission(); + }, []); const closeModal = () => setModalVisible(false); const deleteAndCloseModal = () => { setModalVisible(false); - deleteAnalytics(item); + + if ((userCanDeleteIndex && deleteTargetIndex) || (userCanDeleteIndex && deleteIndexPattern)) { + deleteAnalyticsAndDestIndex( + item, + deleteTargetIndex, + indexPatternExists && deleteIndexPattern + ); + } else { + deleteAnalytics(item); + } }; const openModal = () => setModalVisible(true); + const toggleDeleteIndex = () => setDeleteTargetIndex(!deleteTargetIndex); + const toggleDeleteIndexPattern = () => setDeleteIndexPattern(!deleteIndexPattern); const buttonDeleteText = i18n.translate('xpack.ml.dataframe.analyticsList.deleteActionName', { defaultMessage: 'Delete', @@ -84,8 +174,9 @@ export const DeleteAction: FC = ({ item }) => { {deleteButton} {isModalVisible && ( - + = ({ item }) => { buttonColor="danger" >

- {i18n.translate('xpack.ml.dataframe.analyticsList.deleteModalBody', { - defaultMessage: `Are you sure you want to delete this analytics job? The analytics job's destination index and optional Kibana index pattern will not be deleted.`, - })} +

+ + + + {userCanDeleteIndex && ( + + )} + + + {userCanDeleteIndex && indexPatternExists && ( + + )} + + )} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/delete_analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/delete_analytics.ts index 7383f565bd673a..26cefff0a3f594 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/delete_analytics.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/delete_analytics.ts @@ -3,17 +3,15 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { i18n } from '@kbn/i18n'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { ml } from '../../../../../services/ml_api_service'; - import { refreshAnalyticsList$, REFRESH_ANALYTICS_LIST_STATE } from '../../../../common'; - import { isDataFrameAnalyticsFailed, DataFrameAnalyticsListRow, } from '../../components/analytics_list/common'; +import { extractErrorMessage } from '../../../../../util/error_utils'; export const deleteAnalytics = async (d: DataFrameAnalyticsListRow) => { const toastNotifications = getToastNotifications(); @@ -24,18 +22,139 @@ export const deleteAnalytics = async (d: DataFrameAnalyticsListRow) => { await ml.dataFrameAnalytics.deleteDataFrameAnalytics(d.config.id); toastNotifications.addSuccess( i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsSuccessMessage', { - defaultMessage: 'Request to delete data frame analytics {analyticsId} acknowledged.', + defaultMessage: 'Request to delete data frame analytics job {analyticsId} acknowledged.', values: { analyticsId: d.config.id }, }) ); } catch (e) { + const error = extractErrorMessage(e); + toastNotifications.addDanger( i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsErrorMessage', { defaultMessage: - 'An error occurred deleting the data frame analytics {analyticsId}: {error}', - values: { analyticsId: d.config.id, error: JSON.stringify(e) }, + 'An error occurred deleting the data frame analytics job {analyticsId}: {error}', + values: { analyticsId: d.config.id, error }, }) ); } refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.REFRESH); }; + +export const deleteAnalyticsAndDestIndex = async ( + d: DataFrameAnalyticsListRow, + deleteDestIndex: boolean, + deleteDestIndexPattern: boolean +) => { + const toastNotifications = getToastNotifications(); + const destinationIndex = Array.isArray(d.config.dest.index) + ? d.config.dest.index[0] + : d.config.dest.index; + try { + if (isDataFrameAnalyticsFailed(d.stats.state)) { + await ml.dataFrameAnalytics.stopDataFrameAnalytics(d.config.id, true); + } + const status = await ml.dataFrameAnalytics.deleteDataFrameAnalyticsAndDestIndex( + d.config.id, + deleteDestIndex, + deleteDestIndexPattern + ); + if (status.analyticsJobDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsSuccessMessage', { + defaultMessage: 'Request to delete data frame analytics job {analyticsId} acknowledged.', + values: { analyticsId: d.config.id }, + }) + ); + } + if (status.analyticsJobDeleted?.error) { + const error = extractErrorMessage(status.analyticsJobDeleted.error); + toastNotifications.addDanger( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsErrorMessage', { + defaultMessage: + 'An error occurred deleting the data frame analytics job {analyticsId}: {error}', + values: { analyticsId: d.config.id, error }, + }) + ); + } + + if (status.destIndexDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsWithIndexSuccessMessage', { + defaultMessage: 'Request to delete destination index {destinationIndex} acknowledged.', + values: { destinationIndex }, + }) + ); + } + if (status.destIndexDeleted?.error) { + const error = extractErrorMessage(status.destIndexDeleted.error); + toastNotifications.addDanger( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsWithIndexErrorMessage', { + defaultMessage: + 'An error occurred deleting destination index {destinationIndex}: {error}', + values: { destinationIndex, error }, + }) + ); + } + + if (status.destIndexPatternDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate( + 'xpack.ml.dataframe.analyticsList.deleteAnalyticsWithIndexPatternSuccessMessage', + { + defaultMessage: 'Request to delete index pattern {destinationIndex} acknowledged.', + values: { destinationIndex }, + } + ) + ); + } + if (status.destIndexPatternDeleted?.error) { + const error = extractErrorMessage(status.destIndexPatternDeleted.error); + toastNotifications.addDanger( + i18n.translate( + 'xpack.ml.dataframe.analyticsList.deleteAnalyticsWithIndexPatternErrorMessage', + { + defaultMessage: 'An error occurred deleting index pattern {destinationIndex}: {error}', + values: { destinationIndex, error }, + } + ) + ); + } + } catch (e) { + const error = extractErrorMessage(e); + + toastNotifications.addDanger( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsErrorMessage', { + defaultMessage: + 'An error occurred deleting the data frame analytics job {analyticsId}: {error}', + values: { analyticsId: d.config.id, error }, + }) + ); + } + refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.REFRESH); +}; + +export const canDeleteIndex = async (indexName: string) => { + const toastNotifications = getToastNotifications(); + try { + const privilege = await ml.hasPrivileges({ + index: [ + { + names: [indexName], // uses wildcard + privileges: ['delete_index'], + }, + ], + }); + if (!privilege) { + return false; + } + return privilege.securityDisabled === true || privilege.has_all_requested === true; + } catch (e) { + const error = extractErrorMessage(e); + toastNotifications.addDanger( + i18n.translate('xpack.ml.dataframe.analyticsList.deleteAnalyticsPrivilegeErrorMessage', { + defaultMessage: 'User does not have permission to delete index {indexName}: {error}', + values: { indexName, error }, + }) + ); + } +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/index.ts index 0d1a87e7c4c1f6..68aa58e7e1f198 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/index.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/index.ts @@ -3,8 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - export { getAnalyticsFactory } from './get_analytics'; -export { deleteAnalytics } from './delete_analytics'; +export { deleteAnalytics, deleteAnalyticsAndDestIndex, canDeleteIndex } from './delete_analytics'; export { startAnalytics } from './start_analytics'; export { stopAnalytics } from './stop_analytics'; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts index 89950a659f6099..7cdd5478e39835 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts @@ -10,6 +10,7 @@ import { basePath } from './index'; import { DataFrameAnalyticsStats } from '../../data_frame_analytics/pages/analytics_management/components/analytics_list/common'; import { DataFrameAnalyticsConfig } from '../../data_frame_analytics/common'; import { DeepPartial } from '../../../../common/types/common'; +import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../../../common/types/data_frame_analytics'; export interface GetDataFrameAnalyticsStatsResponseOk { node_failures?: object; @@ -32,6 +33,13 @@ interface GetDataFrameAnalyticsResponse { data_frame_analytics: DataFrameAnalyticsConfig[]; } +interface DeleteDataFrameAnalyticsWithIndexResponse { + acknowledged: boolean; + analyticsJobDeleted: DeleteDataFrameAnalyticsWithIndexStatus; + destIndexDeleted: DeleteDataFrameAnalyticsWithIndexStatus; + destIndexPatternDeleted: DeleteDataFrameAnalyticsWithIndexStatus; +} + export const dataFrameAnalytics = { getDataFrameAnalytics(analyticsId?: string) { const analyticsIdString = analyticsId !== undefined ? `/${analyticsId}` : ''; @@ -86,6 +94,17 @@ export const dataFrameAnalytics = { method: 'DELETE', }); }, + deleteDataFrameAnalyticsAndDestIndex( + analyticsId: string, + deleteDestIndex: boolean, + deleteDestIndexPattern: boolean + ) { + return http({ + path: `${basePath()}/data_frame/analytics/${analyticsId}`, + query: { deleteDestIndex, deleteDestIndexPattern }, + method: 'DELETE', + }); + }, startDataFrameAnalytics(analyticsId: string) { return http({ path: `${basePath()}/data_frame/analytics/${analyticsId}/_start`, diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts index 16e25067fd91e3..e2569f6217b347 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts @@ -95,7 +95,6 @@ export const jobs = { body, }); }, - closeJobs(jobIds: string[]) { const body = JSON.stringify({ jobIds }); return http({ diff --git a/x-pack/plugins/ml/public/application/util/error_utils.ts b/x-pack/plugins/ml/public/application/util/error_utils.ts new file mode 100644 index 00000000000000..2ce8f4ffc583a3 --- /dev/null +++ b/x-pack/plugins/ml/public/application/util/error_utils.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CustomHttpResponseOptions, ResponseError } from 'kibana/server'; + +export const extractErrorMessage = ( + error: CustomHttpResponseOptions | undefined | string +): string | undefined => { + if (typeof error === 'string') { + return error; + } + + if (error?.body) { + if (typeof error.body === 'string') { + return error.body; + } + if (typeof error.body === 'object' && 'message' in error.body) { + if (typeof error.body.message === 'string') { + return error.body.message; + } + // @ts-ignore + if (typeof (error.body.message?.msg === 'string')) { + // @ts-ignore + return error.body.message?.msg; + } + } + } + return undefined; +}; diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts new file mode 100644 index 00000000000000..d1a4df768a6ae5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/index_patterns.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClientContract } from 'kibana/server'; +import { IIndexPattern } from 'src/plugins/data/server'; + +export class IndexPatternHandler { + constructor(private savedObjectsClient: SavedObjectsClientContract) {} + // returns a id based on an index pattern name + async getIndexPatternId(indexName: string) { + const response = await this.savedObjectsClient.find({ + type: 'index-pattern', + perPage: 10, + search: `"${indexName}"`, + searchFields: ['title'], + fields: ['title'], + }); + + const ip = response.saved_objects.find( + (obj) => obj.attributes.title.toLowerCase() === indexName.toLowerCase() + ); + + return ip?.id; + } + + async deleteIndexPatternById(indexId: string) { + return await this.savedObjectsClient.delete('index-pattern', indexId); + } +} diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index 894c4739ef96ef..e2601c7ad6a2e7 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { RequestHandlerContext } from 'kibana/server'; import { wrapError } from '../client/error_wrapper'; import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages'; import { RouteInitialization } from '../types'; @@ -13,12 +14,48 @@ import { dataAnalyticsExplainSchema, analyticsIdSchema, stopsDataFrameAnalyticsJobQuerySchema, + deleteDataFrameAnalyticsJobSchema, } from './schemas/data_analytics_schema'; +import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns'; +import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; + +function getIndexPatternId(context: RequestHandlerContext, patternName: string) { + const iph = new IndexPatternHandler(context.core.savedObjects.client); + return iph.getIndexPatternId(patternName); +} + +function deleteDestIndexPatternById(context: RequestHandlerContext, indexPatternId: string) { + const iph = new IndexPatternHandler(context.core.savedObjects.client); + return iph.deleteIndexPatternById(indexPatternId); +} /** * Routes for the data frame analytics */ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitialization) { + async function userCanDeleteIndex( + context: RequestHandlerContext, + destinationIndex: string + ): Promise { + if (!mlLicense.isSecurityEnabled()) { + return true; + } + const privilege = await context.ml!.mlClient.callAsCurrentUser('ml.privilegeCheck', { + body: { + index: [ + { + names: [destinationIndex], // uses wildcard + privileges: ['delete_index'], + }, + ], + }, + }); + if (!privilege) { + return false; + } + return privilege.has_all_requested === true; + } + /** * @apiGroup DataFrameAnalytics * @@ -277,6 +314,7 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat path: '/api/ml/data_frame/analytics/{analyticsId}', validate: { params: analyticsIdSchema, + query: deleteDataFrameAnalyticsJobSchema, }, options: { tags: ['access:ml:canDeleteDataFrameAnalytics'], @@ -285,12 +323,78 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat mlLicense.fullLicenseAPIGuard(async (context, request, response) => { try { const { analyticsId } = request.params; - const results = await context.ml!.mlClient.callAsCurrentUser( - 'ml.deleteDataFrameAnalytics', - { + const { deleteDestIndex, deleteDestIndexPattern } = request.query; + let destinationIndex: string | undefined; + const analyticsJobDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; + const destIndexDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { success: false }; + const destIndexPatternDeleted: DeleteDataFrameAnalyticsWithIndexStatus = { + success: false, + }; + + // Check if analyticsId is valid and get destination index + if (deleteDestIndex || deleteDestIndexPattern) { + try { + const dfa = await context.ml!.mlClient.callAsCurrentUser('ml.getDataFrameAnalytics', { + analyticsId, + }); + if (Array.isArray(dfa.data_frame_analytics) && dfa.data_frame_analytics.length > 0) { + destinationIndex = dfa.data_frame_analytics[0].dest.index; + } + } catch (e) { + return response.customError(wrapError(e)); + } + + // If user checks box to delete the destinationIndex associated with the job + if (destinationIndex && deleteDestIndex) { + // Verify if user has privilege to delete the destination index + const userCanDeleteDestIndex = await userCanDeleteIndex(context, destinationIndex); + // If user does have privilege to delete the index, then delete the index + if (userCanDeleteDestIndex) { + try { + await context.ml!.mlClient.callAsCurrentUser('indices.delete', { + index: destinationIndex, + }); + destIndexDeleted.success = true; + } catch (deleteIndexError) { + destIndexDeleted.error = wrapError(deleteIndexError); + } + } else { + return response.forbidden(); + } + } + + // Delete the index pattern if there's an index pattern that matches the name of dest index + if (destinationIndex && deleteDestIndexPattern) { + try { + const indexPatternId = await getIndexPatternId(context, destinationIndex); + if (indexPatternId) { + await deleteDestIndexPatternById(context, indexPatternId); + } + destIndexPatternDeleted.success = true; + } catch (deleteDestIndexPatternError) { + destIndexPatternDeleted.error = wrapError(deleteDestIndexPatternError); + } + } + } + // Grab the target index from the data frame analytics job id + // Delete the data frame analytics + + try { + await context.ml!.mlClient.callAsCurrentUser('ml.deleteDataFrameAnalytics', { analyticsId, + }); + analyticsJobDeleted.success = true; + } catch (deleteDFAError) { + analyticsJobDeleted.error = wrapError(deleteDFAError); + if (analyticsJobDeleted.error.statusCode === 404) { + return response.notFound(); } - ); + } + const results = { + analyticsJobDeleted, + destIndexDeleted, + destIndexPatternDeleted, + }; return response.ok({ body: results, }); diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts index f1d4947a7abc5a..0b2469c103578c 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts @@ -60,6 +60,14 @@ export const analyticsIdSchema = schema.object({ analyticsId: schema.string(), }); +export const deleteDataFrameAnalyticsJobSchema = schema.object({ + /** + * Analytics Destination Index + */ + deleteDestIndex: schema.maybe(schema.boolean()), + deleteDestIndexPattern: schema.maybe(schema.boolean()), +}); + export const stopsDataFrameAnalyticsJobQuerySchema = schema.object({ force: schema.maybe(schema.boolean()), }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts new file mode 100644 index 00000000000000..23bff0d0c28550 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { DataFrameAnalyticsConfig } from '../../../../../plugins/ml/public/application/data_frame_analytics/common'; +import { DeepPartial } from '../../../../../plugins/ml/common/types/common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const jobId = `bm_${Date.now()}`; + const generateDestinationIndex = (analyticsId: string) => `user-${analyticsId}`; + const commonJobConfig = { + source: { + index: ['ft_bank_marketing'], + query: { + match_all: {}, + }, + }, + analysis: { + classification: { + dependent_variable: 'y', + training_percent: 20, + }, + }, + analyzed_fields: { + includes: [], + excludes: [], + }, + model_memory_limit: '350mb', + }; + + const testJobConfigs: Array> = [ + 'Test delete job only', + 'Test delete job and target index', + 'Test delete job and index pattern', + 'Test delete job, target index, and index pattern', + ].map((description, idx) => { + const analyticsId = `${jobId}_${idx + 1}`; + return { + id: analyticsId, + description, + dest: { + index: generateDestinationIndex(analyticsId), + results_field: 'ml', + }, + ...commonJobConfig, + }; + }); + + async function createJobs(mockJobConfigs: Array>) { + for (const jobConfig of mockJobConfigs) { + await ml.api.createDataFrameAnalyticsJob(jobConfig as DataFrameAnalyticsConfig); + } + } + + describe('DELETE data_frame/analytics', () => { + before(async () => { + await esArchiver.loadIfNeeded('ml/bm_classification'); + await ml.testResources.setKibanaTimeZoneToUTC(); + await createJobs(testJobConfigs); + }); + + after(async () => { + await ml.api.cleanMlIndices(); + }); + + describe('DeleteDataFrameAnalytics', () => { + it('should delete analytics jobs by id', async () => { + const analyticsId = `${jobId}_1`; + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.analyticsJobDeleted.success).to.eql(true); + await ml.api.waitForDataFrameAnalyticsJobNotToExist(analyticsId); + }); + + it('should not allow to retrieve analytics jobs for unauthorized user', async () => { + const analyticsId = `${jobId}_2`; + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED)) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.eql('Not Found'); + expect(body.message).to.eql('Not Found'); + await ml.api.waitForDataFrameAnalyticsJobToExist(analyticsId); + }); + + it('should not allow to retrieve analytics jobs for the user with only view permission', async () => { + const analyticsId = `${jobId}_2`; + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.eql('Not Found'); + expect(body.message).to.eql('Not Found'); + await ml.api.waitForDataFrameAnalyticsJobToExist(analyticsId); + }); + + it('should show 404 error if job does not exist or has already been deleted', async () => { + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${jobId}_invalid`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.eql('Not Found'); + expect(body.message).to.eql('Not Found'); + }); + + describe('with deleteDestIndex setting', function () { + const analyticsId = `${jobId}_2`; + const destinationIndex = generateDestinationIndex(analyticsId); + + before(async () => { + await ml.api.createIndices(destinationIndex); + await ml.api.assertIndicesExist(destinationIndex); + }); + + after(async () => { + await ml.api.deleteIndices(destinationIndex); + }); + + it('should delete job and destination index by id', async () => { + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .query({ deleteDestIndex: true }) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.analyticsJobDeleted.success).to.eql(true); + expect(body.destIndexDeleted.success).to.eql(true); + expect(body.destIndexPatternDeleted.success).to.eql(false); + await ml.api.waitForDataFrameAnalyticsJobNotToExist(analyticsId); + await ml.api.assertIndicesNotToExist(destinationIndex); + }); + }); + + describe('with deleteDestIndexPattern setting', function () { + const analyticsId = `${jobId}_3`; + const destinationIndex = generateDestinationIndex(analyticsId); + + before(async () => { + // Mimic real job by creating index pattern after job is created + await ml.testResources.createIndexPatternIfNeeded(destinationIndex); + }); + + after(async () => { + await ml.testResources.deleteIndexPattern(destinationIndex); + }); + + it('should delete job and index pattern by id', async () => { + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .query({ deleteDestIndexPattern: true }) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.analyticsJobDeleted.success).to.eql(true); + expect(body.destIndexDeleted.success).to.eql(false); + expect(body.destIndexPatternDeleted.success).to.eql(true); + await ml.api.waitForDataFrameAnalyticsJobNotToExist(analyticsId); + await ml.testResources.assertIndexPatternNotExist(destinationIndex); + }); + }); + + describe('with deleteDestIndex & deleteDestIndexPattern setting', function () { + const analyticsId = `${jobId}_4`; + const destinationIndex = generateDestinationIndex(analyticsId); + + before(async () => { + // Mimic real job by creating target index & index pattern after DFA job is created + await ml.api.createIndices(destinationIndex); + await ml.api.assertIndicesExist(destinationIndex); + await ml.testResources.createIndexPatternIfNeeded(destinationIndex); + }); + + after(async () => { + await ml.api.deleteIndices(destinationIndex); + await ml.testResources.deleteIndexPattern(destinationIndex); + }); + + it('deletes job, target index, and index pattern by id', async () => { + const { body } = await supertest + .delete(`/api/ml/data_frame/analytics/${analyticsId}`) + .query({ deleteDestIndex: true, deleteDestIndexPattern: true }) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.analyticsJobDeleted.success).to.eql(true); + expect(body.destIndexDeleted.success).to.eql(true); + expect(body.destIndexPatternDeleted.success).to.eql(true); + await ml.api.waitForDataFrameAnalyticsJobNotToExist(analyticsId); + await ml.api.assertIndicesNotToExist(destinationIndex); + await ml.testResources.assertIndexPatternNotExist(destinationIndex); + }); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/index.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/index.ts index 9e0f952ad501ba..6693561076fdd6 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/index.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/index.ts @@ -9,5 +9,6 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('data frame analytics', function () { loadTestFile(require.resolve('./get')); + loadTestFile(require.resolve('./delete')); }); } diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index 897f37821001e1..fc2ce4bb16b99f 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -9,9 +9,9 @@ import { DataFrameAnalyticsConfig } from '../../../../plugins/ml/public/applicat import { FtrProviderContext } from '../../ftr_provider_context'; -import { JOB_STATE, DATAFEED_STATE } from '../../../../plugins/ml/common/constants/states'; +import { DATAFEED_STATE, JOB_STATE } from '../../../../plugins/ml/common/constants/states'; import { DATA_FRAME_TASK_STATE } from '../../../../plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/common'; -import { Job, Datafeed } from '../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import { Datafeed, Job } from '../../../../plugins/ml/common/types/anomaly_detection_jobs'; export type MlApi = ProvidedType; @@ -110,6 +110,21 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { ); }, + async createIndices(indices: string) { + log.debug(`Creating indices: '${indices}'...`); + if ((await es.indices.exists({ index: indices, allowNoIndices: false })) === true) { + log.debug(`Indices '${indices}' already exist. Nothing to create.`); + return; + } + + const createResponse = await es.indices.create({ index: indices }); + expect(createResponse) + .to.have.property('acknowledged') + .eql(true, 'Response for create request indices should be acknowledged.'); + + await this.assertIndicesExist(indices); + }, + async deleteIndices(indices: string) { log.debug(`Deleting indices: '${indices}'...`); if ((await es.indices.exists({ index: indices, allowNoIndices: false })) === false) { @@ -122,15 +137,9 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { }); expect(deleteResponse) .to.have.property('acknowledged') - .eql(true, 'Response for delete request should be acknowledged'); + .eql(true, 'Response for delete request should be acknowledged.'); - await retry.waitForWithTimeout(`'${indices}' indices to be deleted`, 30 * 1000, async () => { - if ((await es.indices.exists({ index: indices, allowNoIndices: false })) === false) { - return true; - } else { - throw new Error(`expected indices '${indices}' to be deleted`); - } - }); + await this.assertIndicesNotToExist(indices); }, async cleanMlIndices() { @@ -251,6 +260,16 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { }); }, + async assertIndicesNotToExist(indices: string) { + await retry.tryForTime(30 * 1000, async () => { + if ((await es.indices.exists({ index: indices, allowNoIndices: false })) === false) { + return true; + } else { + throw new Error(`indices '${indices}' should not exist`); + } + }); + }, + async assertIndicesNotEmpty(indices: string) { await retry.tryForTime(30 * 1000, async () => { const response = await es.search({ @@ -394,9 +413,9 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { await this.waitForJobState(jobConfig.job_id, JOB_STATE.CLOSED); }, - async getDataFrameAnalyticsJob(analyticsId: string) { + async getDataFrameAnalyticsJob(analyticsId: string, statusCode = 200) { log.debug(`Fetching data frame analytics job '${analyticsId}'...`); - return await esSupertest.get(`/_ml/data_frame/analytics/${analyticsId}`).expect(200); + return await esSupertest.get(`/_ml/data_frame/analytics/${analyticsId}`).expect(statusCode); }, async waitForDataFrameAnalyticsJobToExist(analyticsId: string) { @@ -409,6 +428,16 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { }); }, + async waitForDataFrameAnalyticsJobNotToExist(analyticsId: string) { + await retry.waitForWithTimeout(`'${analyticsId}' not to exist`, 5 * 1000, async () => { + if (await this.getDataFrameAnalyticsJob(analyticsId, 404)) { + return true; + } else { + throw new Error(`expected data frame analytics job '${analyticsId}' not to exist`); + } + }); + }, + async createDataFrameAnalyticsJob(jobConfig: DataFrameAnalyticsConfig) { const { id: analyticsId, ...analyticsConfig } = jobConfig; log.debug(`Creating data frame analytic job with id '${analyticsId}'...`); diff --git a/x-pack/test/functional/services/ml/test_resources.ts b/x-pack/test/functional/services/ml/test_resources.ts index d349416ec90f7c..739fd844f11933 100644 --- a/x-pack/test/functional/services/ml/test_resources.ts +++ b/x-pack/test/functional/services/ml/test_resources.ts @@ -5,7 +5,6 @@ */ import { ProvidedType } from '@kbn/test/types/ftr'; - import { savedSearches } from './test_resources_data'; import { COMMON_REQUEST_HEADERS } from './common'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -24,6 +23,7 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider const kibanaServer = getService('kibanaServer'); const log = getService('log'); const supertest = getService('supertest'); + const retry = getService('retry'); return { async setKibanaTimeZoneToUTC() { @@ -98,6 +98,21 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider } }, + async assertIndexPatternNotExist(title: string) { + await retry.waitForWithTimeout( + `index pattern '${title}' to not exist`, + 5 * 1000, + async () => { + const indexPatternId = await this.getIndexPatternId(title); + if (!indexPatternId) { + return true; + } else { + throw new Error(`Index pattern '${title}' should not exist.`); + } + } + ); + }, + async createSavedSearch(title: string, body: object): Promise { log.debug(`Creating saved search with title '${title}'`);